From e9f8ff2506b2d15cf51cc2de08a23de1bc1c7bff Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Fri, 1 Jul 2022 14:56:41 +0200 Subject: [PATCH] fix(bootstraper): adding global variables support Now we can declare "global" variables, in the "Globals" section the the configuration, this variables will be merged into the "Service" variables and used in the templates. This how we share variables between services. --- cmd/bootstraper/main.go | 4 +- data/config/loki-stack.json | 204 ++++ data/templates/alertmanager.yml.pktpl.hcl | 16 + data/templates/grafana.ini.pktpl.hcl | 1160 +++++++++++++++++++- data/templates/loki-local-config.pktpl.hcl | 4 +- data/templates/loki-local-config.tpl | 30 +- data/templates/prometheus.yml.pktpl.hcl | 34 + pkg/templater/main.go | 25 +- 8 files changed, 1447 insertions(+), 30 deletions(-) create mode 100644 data/config/loki-stack.json create mode 100644 data/templates/alertmanager.yml.pktpl.hcl create mode 100644 data/templates/prometheus.yml.pktpl.hcl diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go index 68f0732..9701963 100644 --- a/cmd/bootstraper/main.go +++ b/cmd/bootstraper/main.go @@ -17,11 +17,11 @@ func main() { var hostConfig templater.TemplaterConfig - err := hostConfig.New(args.Config, args.TemplateDirectory) + err := hostConfig.New(args.Config, args.TemplateDirectory, args.RootDirectory) if err != nil { panic(err) } - if err = hostConfig.ManageServices(args.RootDirectory); err != nil { + if err = hostConfig.ManageServices(); err != nil { panic(err) } } diff --git a/data/config/loki-stack.json b/data/config/loki-stack.json new file mode 100644 index 0000000..db657ea --- /dev/null +++ b/data/config/loki-stack.json @@ -0,0 +1,204 @@ +{ + "Globals": { + "Vars": { + "PrometheusPort": "9090" + } + }, + "Name": "loki-stack", + "Services": { + "Alertmanager": { + "ConfigFiles": [ + { + "destination": "/etc/alertmanager/alertmanager.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "alertmanager.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "alertmanager", + "type": "auto" + } + }, + "Packages": { + "alertmanager": { + "action": "install", + "name": "alertmanager" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": {} + }, + "Grafana": { + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "group": "grafana", + "mode": "600", + "owner": "grafana", + "source": "grafana.ini.pktpl.hcl" + } + ], + "Daemons": { + "grafana": { + "enabled": true, + "name": "grafana", + "type": "auto" + } + }, + "Packages": { + "grafana": { + "action": "install", + "name": "grafana" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "grafana": { + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin", + "username": "grafana" + } + }, + "Vars": { + "AppMode": "production", + "DomainName": "www.grafana.local", + "HTTPPort": "80", + "HostName": "grafana.local", + "UserName": "grafana" + } + }, + "Loki": { + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "group": "grafana", + "mode": "600", + "owner": "loki", + "service": "loki", + "source": "loki-local-config.pktpl.hcl" + } + ], + "Daemons": { + "Loki": { + "enabled": true, + "name": "loki" + } + }, + "Packages": { + "loki": { + "action": "install", + "name": "loki" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "promtail": { + "action": "install", + "name": "loki-promtail" + } + }, + "Repositories": { + "AlpineTesting": { + "enabled": true, + "name": "testing", + "type": "apk", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing" + } + }, + "Users": { + "loki": { + "group": "grafana", + "home": "/srv/loki", + "shell": "/bin/nologin", + "username": "loki" + } + }, + "Vars": { + "AlertManagerURL": "http://localhost:9092", + "AuthEnabled": false, + "GRPCPort": "9095", + "Group": "grafana", + "HTTPPort": "3099", + "LogLevel": "error", + "ObjectStore": "filesystem", + "S2": { + "APIKey": "", + "APISecretKey": "", + "BucketName": "", + "URL": "" + }, + "SharedStore": "filesystem", + "StorageRoot": "/var/loki", + "User": "loki" + } + }, + "Prometheus": { + "ConfigFiles": [ + { + "destination": "/etc/prometheus/prometheus.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "prometheus.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "prometheus", + "type": "auto" + } + }, + "Packages": { + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "prometheus": { + "action": "install", + "name": "prometheus" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": { + "Scrapers": [ + { + "MetricsPath": "/metrics", + "Name": "Prometheus", + "Scheme": "http", + "Targets": [ + "localhost:9001" + ] + } + ] + } + } + } +} diff --git a/data/templates/alertmanager.yml.pktpl.hcl b/data/templates/alertmanager.yml.pktpl.hcl new file mode 100644 index 0000000..7bd5e0c --- /dev/null +++ b/data/templates/alertmanager.yml.pktpl.hcl @@ -0,0 +1,16 @@ +route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 1h + receiver: 'web.hook' +receivers: + - name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:5001/' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl index 3ba92f6..8b42f6d 100644 --- a/data/templates/grafana.ini.pktpl.hcl +++ b/data/templates/grafana.ini.pktpl.hcl @@ -1,5 +1,1155 @@ -%{ if Vars.AuthEnabled ~} -auth_enabled: true -%{ else } -auth_enabled: false -%{ endif } \ No newline at end of file +#####################.Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +app_mode = ${Vars.AppMode} + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${Vars.HostName} + +# force migration will run migrations that might cause dataloss +;force_migration = false + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +http_port = ${Vars.HTTPPort} + +# The public facing domain name used to access grafana from a browser +domain = ${Vars.DomainName} + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve.from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = ${Vars.UserName} +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "mysql" only if lockingMigration feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that.will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use.primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that.will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X.User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that.will process from SQL data sources. +;row_limit = 1000000 + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the.instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host.behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render.in a ,