diff --git a/.gitignore b/.gitignore index f4d432a..5bbb8bf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ # Dependency directories (remove the comment below to include it) # vendor/ - +bin/ diff --git a/Makefile b/Makefile index b2d52ac..a8b6c47 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,14 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux main.go + GOOS=linux CGO_ENABLED=0 go build -o bin/templater-linux cmd/templater/main.go + GOOS=linux CGO_ENABLED=0 go build -o bin/bootstraper-linux cmd/bootstraper/main.go upx bin/templater-linux + upx bin/bootstraper-linux install: cp bin/templater-linux $(DESTDIR)/bin/templater + cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper uninstall: rm $(DESTDIR)/bin/templater diff --git a/api/main.go b/api/main.go new file mode 100644 index 0000000..d0766f1 --- /dev/null +++ b/api/main.go @@ -0,0 +1,48 @@ +package api + +import ( + "github.com/gin-gonic/gin" +) + +type Template struct { + Type string + Content string + Config string +} + +func Generate(c *gin.Context) { + return +} + +/* + +func Generate(c *gin.Context) { + var template Template + + err := c.Request.ParseForm() + if err != nil { + c.String(500, err.Error()) + } + + err = c.ShouldBindJSON(&template) + if err != nil { + c.String(500, err.Error()) + return + } + + templateType := template.Type + templateFile := template.Content + config := []byte(template.Config) + res := "" + if templateType == "go" { + res = templater.ProcessGoTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else if templateType == "hcl" { + res = templater.ProcessHCLTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else { + c.JSON(http.StatusBadRequest, gin.H{"data": "Unkown template type"}) + } + +} +*/ diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go new file mode 100644 index 0000000..1937886 --- /dev/null +++ b/cmd/bootstraper/main.go @@ -0,0 +1,28 @@ +package main + +import ( + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" +) + +func main() { + + var args struct { + Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"` + TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"` + RootDirectory string `arg:"-r,--root-dir,env:ROOT_DIR" help:"Generate files with this root instead of /" default:"/"` + DryRun bool `arg:"-d,--dry-run,env:DRY_RUN" help:"Dry run do not really complete actions" default:"false"` + } + + arg.MustParse(&args) + + var hostConfig templater.TemplaterConfig + + err := hostConfig.New(args.Config, args.TemplateDirectory, args.RootDirectory) + if err != nil { + panic(err) + } + if err = hostConfig.ManageServices(args.DryRun); err != nil { + panic(err) + } +} diff --git a/cmd/templater/main.go b/cmd/templater/main.go new file mode 100644 index 0000000..5fba371 --- /dev/null +++ b/cmd/templater/main.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/api" + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" + "github.com/gin-gonic/gin" +) + +func Daemon(port int) (err error) { + r := gin.Default() + + r.POST("/generate", api.Generate) + + err = r.Run(fmt.Sprintf("0.0.0.0:%d", port)) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") + if err != nil { + return (err) + } + return nil +} + +func main() { + + var args struct { + Daemon bool `arg:"-d,--daemon,env:TEMPLATER_DAEMON" default:"false" help:"Enable api server"` + Port int `arg:"-p,--port,env:TEMPLATER_PORT" default:"8080" help:"Listening port for the api server"` + Type string `arg:"-t,--type,env:TEMPLATE_TYPE" default:"hcl" help:"Template type (go/template or hcl)"` + Output string `arg:"-o,--output,env:TEMPLATER_OUTPUT" default:"stdout" help:"Destination of the result (stdout or file path)"` + Config string `arg:"-c,--config,env:TEMPLATE_CONFIG" help:"Configuration values"` + File string `arg:"-f,--template-file,env:TEMPLATE_FILE" help:"Template file path"` + } + + arg.MustParse(&args) + + if args.Daemon { + err := Daemon(args.Port) + if err != nil { + panic(err) + } + } else { + var config []byte + templateType := args.Type + templateFile := args.File + output := args.Output + + if _, err := os.Stat(args.Config); err == nil { + config, err = os.ReadFile(args.Config) + if err != nil { + panic(err) + } + } else { + config = []byte(args.Config) + } + + var file templater.ConfigFile + file.Source = templateFile + file.TemplateType = templateType + + result, err := file.ProcessTemplate(templateFile, config) + if err != nil { + panic(err) + } + if output == "stdout" { + fmt.Printf("%s", result) + } else { + err := os.WriteFile(output, []byte(result), 0644) + if err != nil { + panic(err) + } + } + } + +} diff --git a/data/config/go-test-conf.json b/data/config/go-test-conf.json new file mode 100644 index 0000000..bddc09d --- /dev/null +++ b/data/config/go-test-conf.json @@ -0,0 +1,66 @@ +{ + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "group": "grafana", + "mode": "600", + "owner": "loki", + "service": "loki", + "source": "loki-local-config.pktpl.hcl" + } + ], + "Daemons": { + "Loki": { + "enabled": true, + "name": "loki" + } + }, + "Packages": { + "loki": { + "action": "install", + "name": "loki" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "promtail": { + "action": "install", + "name": "loki-promtail" + } + }, + "Repositories": { + "AlpineTesting": { + "enabled": true, + "name": "testing", + "type": "apk", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing" + } + }, + "Users": { + "loki": { + "group": "grafana", + "home": "/srv/loki", + "shell": "/bin/nologin", + "username": "loki" + } + }, + "Vars": { + "AlertManagerURL": "http://localhost:9092", + "AuthEnabled": false, + "GRPCPort": "9095", + "Group": "grafana", + "HTTPPort": "3099", + "LogLevel": "error", + "ObjectStore": "filesystem", + "S2": { + "APIKey": "", + "APISecretKey": "", + "BucketName": "", + "URL": "" + }, + "SharedStore": "filesystem", + "StorageRoot": "/var/loki", + "User": "loki" + } + } \ No newline at end of file diff --git a/data/config/loki-stack.json b/data/config/loki-stack.json new file mode 100644 index 0000000..db657ea --- /dev/null +++ b/data/config/loki-stack.json @@ -0,0 +1,204 @@ +{ + "Globals": { + "Vars": { + "PrometheusPort": "9090" + } + }, + "Name": "loki-stack", + "Services": { + "Alertmanager": { + "ConfigFiles": [ + { + "destination": "/etc/alertmanager/alertmanager.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "alertmanager.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "alertmanager", + "type": "auto" + } + }, + "Packages": { + "alertmanager": { + "action": "install", + "name": "alertmanager" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": {} + }, + "Grafana": { + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "group": "grafana", + "mode": "600", + "owner": "grafana", + "source": "grafana.ini.pktpl.hcl" + } + ], + "Daemons": { + "grafana": { + "enabled": true, + "name": "grafana", + "type": "auto" + } + }, + "Packages": { + "grafana": { + "action": "install", + "name": "grafana" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "grafana": { + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin", + "username": "grafana" + } + }, + "Vars": { + "AppMode": "production", + "DomainName": "www.grafana.local", + "HTTPPort": "80", + "HostName": "grafana.local", + "UserName": "grafana" + } + }, + "Loki": { + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "group": "grafana", + "mode": "600", + "owner": "loki", + "service": "loki", + "source": "loki-local-config.pktpl.hcl" + } + ], + "Daemons": { + "Loki": { + "enabled": true, + "name": "loki" + } + }, + "Packages": { + "loki": { + "action": "install", + "name": "loki" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "promtail": { + "action": "install", + "name": "loki-promtail" + } + }, + "Repositories": { + "AlpineTesting": { + "enabled": true, + "name": "testing", + "type": "apk", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing" + } + }, + "Users": { + "loki": { + "group": "grafana", + "home": "/srv/loki", + "shell": "/bin/nologin", + "username": "loki" + } + }, + "Vars": { + "AlertManagerURL": "http://localhost:9092", + "AuthEnabled": false, + "GRPCPort": "9095", + "Group": "grafana", + "HTTPPort": "3099", + "LogLevel": "error", + "ObjectStore": "filesystem", + "S2": { + "APIKey": "", + "APISecretKey": "", + "BucketName": "", + "URL": "" + }, + "SharedStore": "filesystem", + "StorageRoot": "/var/loki", + "User": "loki" + } + }, + "Prometheus": { + "ConfigFiles": [ + { + "destination": "/etc/prometheus/prometheus.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "prometheus.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "prometheus", + "type": "auto" + } + }, + "Packages": { + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "prometheus": { + "action": "install", + "name": "prometheus" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": { + "Scrapers": [ + { + "MetricsPath": "/metrics", + "Name": "Prometheus", + "Scheme": "http", + "Targets": [ + "localhost:9001" + ] + } + ] + } + } + } +} diff --git a/data/schema/templater.hcl b/data/schema/templater.hcl new file mode 100644 index 0000000..e69de29 diff --git a/data/schema/templater.json b/data/schema/templater.json new file mode 100644 index 0000000..e69de29 diff --git a/data/templates/alertmanager.yml.pktpl.hcl b/data/templates/alertmanager.yml.pktpl.hcl new file mode 100644 index 0000000..7bd5e0c --- /dev/null +++ b/data/templates/alertmanager.yml.pktpl.hcl @@ -0,0 +1,16 @@ +route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 1h + receiver: 'web.hook' +receivers: + - name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:5001/' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] diff --git a/data/templates/go-test-go.tpl b/data/templates/go-test-go.tpl new file mode 100644 index 0000000..953cb01 --- /dev/null +++ b/data/templates/go-test-go.tpl @@ -0,0 +1,13 @@ +### Go template test ### +{{ if .Vars.AuthEnabled }} +auth_enabled: true +{{ else }} +auth_enabled: false +{{ end }} + +server: + http_listen_port: {{ .Vars.HTTPPort }} + grpc_listen_port: {{ .Vars.GRPCPort }} + log_level: {{ .Vars.LogLevel }} + +### End Go template test ### \ No newline at end of file diff --git a/data/templates/go-test-hcl.pktpl.hcl b/data/templates/go-test-hcl.pktpl.hcl new file mode 100644 index 0000000..1671af5 --- /dev/null +++ b/data/templates/go-test-hcl.pktpl.hcl @@ -0,0 +1,12 @@ +### HCL2 Template test ### +%{ if Vars.AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } + +server: + http_listen_port: ${Vars.HTTPPort} + grpc_listen_port: ${Vars.GRPCPort} + log_level: ${Vars.LogLevel} +### END HCL Template test ### diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl new file mode 100644 index 0000000..8b42f6d --- /dev/null +++ b/data/templates/grafana.ini.pktpl.hcl @@ -0,0 +1,1155 @@ +#####################.Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +app_mode = ${Vars.AppMode} + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${Vars.HostName} + +# force migration will run migrations that might cause dataloss +;force_migration = false + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +http_port = ${Vars.HTTPPort} + +# The public facing domain name used to access grafana from a browser +domain = ${Vars.DomainName} + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve.from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = ${Vars.UserName} +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "mysql" only if lockingMigration feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that.will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use.primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that.will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X.User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that.will process from SQL data sources. +;row_limit = 1000000 + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the.instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host.behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render.in a ,