From 54ef86c46effd3a9b9ea7291c407d6b2383c9d24 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Fri, 11 Mar 2022 10:39:07 +0100 Subject: [PATCH 1/9] fix(template): fix go txt/template support We can parse go text/template again. --- main.go | 54 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index 8e16cbd..b32edde 100644 --- a/main.go +++ b/main.go @@ -1,15 +1,14 @@ package main import ( + encjson "encoding/json" "fmt" "os" - "encoding/json" "text/template" - //"github.com/hashicorp/hcl2/gohcl" + "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" - //"github.com/zclconf/go-cty/cty" - //"github.com/zclconf/go-cty/cty/gocty" + "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" ) @@ -19,40 +18,58 @@ func checkErr(e error) { } } +func checkDiags(diag hcl.Diagnostics) { + if diag.HasErrors() { + panic(diag.Error()) + } +} + func processGoTemplate(file string, config []byte) { // The JSON configuration var confData map[string]interface{} - err := json.Unmarshal(config, &confData) + err := encjson.Unmarshal(config, &confData) checkErr(err) // Read the template data, err := os.ReadFile(file) checkErr(err) - tpl, err := template.New("conf").Parse(string(data)) checkErr(err) - checkErr(tpl.Execute(os.Stdout,config)) + checkErr(tpl.Execute(os.Stdout, confData)) } func processHCLTemplate(file string, config []byte) { - fct, err:= os.ReadFile(file) + fct, err := os.ReadFile(file) checkErr(err) - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line:1, Column: 1}) - if diags.HasErrors() { - panic(diags.Error()) + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1}) + checkDiags(diags) + + // Retrieve values from JSON + var varsVal cty.Value + ctyType, err := ctyjson.ImpliedType(config) + if err != nil { + panic(err) + /* Maybe one day + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1}) + if diags.HasErrors() { + panic(diags.Error()) + } + varsVal, diags = cexpr.Value(&hcl.EvalContext{}) + fmt.Println(cexpr.Variables()) + checkDiags(diags) + */ + } else { + varsVal, err = ctyjson.Unmarshal(config, ctyType) + checkErr(err) } - ctyType, err := ctyjson.ImpliedType(config) - checkErr(err) - varsVal, err := ctyjson.Unmarshal(config,ctyType) - checkErr(err) - + fmt.Println(varsVal.AsValueMap()) ctx := &hcl.EvalContext{ Variables: varsVal.AsValueMap(), } @@ -75,7 +92,7 @@ func processHCLTemplate(file string, config []byte) { panic(diags.Error()) } - fmt.Printf("%s",val.AsString()) + fmt.Printf("%s", val.AsString()) } func main() { @@ -84,7 +101,6 @@ func main() { templateFile := os.Args[2] config := []byte(os.Args[3]) - if templateType == "go" { processGoTemplate(templateFile, config) } else if templateType == "hcl" { @@ -92,4 +108,4 @@ func main() { } else { panic(fmt.Errorf("Unsupported template type")) } -} \ No newline at end of file +} From b22cbcaf786660288a3214dbf9e8bec9adce24d2 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Mon, 14 Mar 2022 14:09:19 +0100 Subject: [PATCH 2/9] Remove unwanted output --- main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/main.go b/main.go index b32edde..046fe41 100644 --- a/main.go +++ b/main.go @@ -69,7 +69,6 @@ func processHCLTemplate(file string, config []byte) { checkErr(err) } - fmt.Println(varsVal.AsValueMap()) ctx := &hcl.EvalContext{ Variables: varsVal.AsValueMap(), } From 67e8485958cf9b53d3f3fbe229bf09efbd30202d Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Thu, 24 Mar 2022 00:49:23 +0100 Subject: [PATCH 3/9] feat(api): adding api server code templater can be used as a commmand line tool or as an api server. --- Makefile | 3 +- api/main.go | 44 ++++++++++++ cmd/templater.go | 76 ++++++++++++++++++++ data/config/test.json | 26 +++++++ data/templates/loki-local-config.pktpl.hcl | 82 ++++++++++++++++++++++ data/templates/loki-local-config.tpl | 82 ++++++++++++++++++++++ main.go => pkg/templater/main.go | 59 +++++----------- pkg/utils/main.go | 17 +++++ 8 files changed, 348 insertions(+), 41 deletions(-) create mode 100644 api/main.go create mode 100644 cmd/templater.go create mode 100644 data/config/test.json create mode 100644 data/templates/loki-local-config.pktpl.hcl create mode 100644 data/templates/loki-local-config.tpl rename main.go => pkg/templater/main.go (66%) create mode 100644 pkg/utils/main.go diff --git a/Makefile b/Makefile index b2d52ac..254cad2 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux main.go + GOOS=linux go build -o bin/templater-linux cmd/templater.go upx bin/templater-linux + upx bin/templaster-server install: cp bin/templater-linux $(DESTDIR)/bin/templater diff --git a/api/main.go b/api/main.go new file mode 100644 index 0000000..7c12c62 --- /dev/null +++ b/api/main.go @@ -0,0 +1,44 @@ +package api + +import ( + "net/http" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/gin-gonic/gin" +) + +type Template struct { + Type string + Content string + Config string +} + +func Generate(c *gin.Context) { + var template Template + + err := c.Request.ParseForm() + if err != nil { + c.String(500, err.Error()) + } + + err = c.ShouldBindJSON(&template) + if err != nil { + c.String(500, err.Error()) + return + } + + templateType := template.Type + templateFile := template.Content + config := []byte(template.Config) + res := "" + if templateType == "go" { + res = templater.ProcessGoTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else if templateType == "hcl" { + res = templater.ProcessHCLTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else { + c.JSON(http.StatusBadRequest, gin.H{"data": "Unkown template type"}) + } + +} diff --git a/cmd/templater.go b/cmd/templater.go new file mode 100644 index 0000000..904353e --- /dev/null +++ b/cmd/templater.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/api" + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" + "github.com/gin-gonic/gin" +) + +func Daemon(port int) (err error) { + r := gin.Default() + + r.POST("/generate", api.Generate) + + err = r.Run(fmt.Sprintf("0.0.0.0:%d", port)) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") + if err != nil { + return (err) + } + return nil +} + +func main() { + + var args struct { + Daemon bool `arg:"-d,--daemon,env:TEMPLATER_DAEMON" default:"false" help:"Enable api server"` + Port int `arg:"-p,--port,env:TEMPLATER_PORT" default:"8080" help:"Listening port for the api server"` + Type string `arg:"-t,--type,env:TEMPLATE_TYPE" default:"hcl" help:"Template type (go/template or hcl)"` + Output string `arg:"-o,--output,env:TEMPLATER_OUTPUT" default:"stdout" help:"Destination of the result (stdout or file path)"` + Config string `arg:"-c,--config,env:TEMPLATE_CONFIG" help:"Configuration values"` + File string `arg:"-f,--template-file,env:TEMPLATE_FILE" help:"Template file path"` + } + + arg.MustParse(&args) + + if args.Daemon { + err := Daemon(args.Port) + if err != nil { + panic(err) + } + } else { + var config []byte + templateType := args.Type + templateFile := args.File + output := args.Output + + if _, err := os.Stat(args.Config); err == nil { + config, err = os.ReadFile(args.Config) + if err != nil { + panic(err) + } + } else { + config = []byte(args.Config) + } + + result := "" + if templateType == "go" { + result = templater.ProcessGoTemplate(templateFile, config) + } else if templateType == "hcl" { + result = templater.ProcessHCLTemplate(templateFile, config) + } else { + panic(fmt.Errorf("Unsupported template type")) + } + if output == "stdout" { + fmt.Printf("%s", result) + } else { + err := os.WriteFile(output, []byte(result), 0644) + if err != nil { + panic(err) + } + } + } + +} diff --git a/data/config/test.json b/data/config/test.json new file mode 100644 index 0000000..14035cb --- /dev/null +++ b/data/config/test.json @@ -0,0 +1,26 @@ +{ + "Name": "loki", + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mod": "600" + } + ], + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } +} \ No newline at end of file diff --git a/data/templates/loki-local-config.pktpl.hcl b/data/templates/loki-local-config.pktpl.hcl new file mode 100644 index 0000000..9fd10f7 --- /dev/null +++ b/data/templates/loki-local-config.pktpl.hcl @@ -0,0 +1,82 @@ +%{ if AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } + +server: + http_listen_port: ${HTTPPort} + grpc_listen_port: ${GRPCPort} + log_level: ${LogLevel} + +ingester: + wal: + enabled: true + dir: ${StorageRoot}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: ${ObjectStore} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: ${StorageRoot}/index + shared_store: ${SharedStore} + cache_location: ${StorageRoot}/cache + cache_ttl: 168h + +%{ if ObjectStore == "filesystem" ~} + filesystem: + directory: ${StorageRoot}/chunks +%{ else } + aws: + s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName} + s3forcepathstyle: true +%{ endif } + +compactor: + shared_store: ${SharedStore} + working_directory: ${StorageRoot}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: ${StorageRoot}/rules + rule_path: ${StorageRoot}/rules + alertmanager_url: ${AlertManagerURL} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl new file mode 100644 index 0000000..07cd8c8 --- /dev/null +++ b/data/templates/loki-local-config.tpl @@ -0,0 +1,82 @@ +{{ if .AuthEnabled }} +auth_enabled: true +{{ else }} +auth_enabled: false +{{ end }} + +server: + http_listen_port: {{ .HTTPPort }} + grpc_listen_port: {{ .GRPCPort }} + log_level: {{ .LogLevel }} + +ingester: + wal: + enabled: true + dir: {{ .StorageRoot }}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: {{ .ObjectStore }} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: {{ .StorageRoot }}/index + shared_store: {{ .SharedStore }} + cache_location: {{ .StorageRoot }}/cache + cache_ttl: 168h + +{{ if eq (.ObjectStore) ("filesystem") }} + filesystem: + directory: {{ .StorageRoot }}/chunks +{{ else }} + aws: + s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} + s3forcepathstyle: true +{{ end }} + +compactor: + shared_store: {{ .SharedStore }} + working_directory: {{ .StorageRoot }}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: {{ .StorageRoot }}/rules + rule_path: {{ .StorageRoot }}/rules + alertmanager_url: {{ .AlertManagerURL }} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/main.go b/pkg/templater/main.go similarity index 66% rename from main.go rename to pkg/templater/main.go index 046fe41..f1fba18 100644 --- a/main.go +++ b/pkg/templater/main.go @@ -1,6 +1,7 @@ -package main +package templater import ( + "bytes" encjson "encoding/json" "fmt" "os" @@ -10,45 +11,38 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" ) -func checkErr(e error) { - if e != nil { - panic(e) - } -} - -func checkDiags(diag hcl.Diagnostics) { - if diag.HasErrors() { - panic(diag.Error()) - } -} - -func processGoTemplate(file string, config []byte) { +func ProcessGoTemplate(file string, config []byte) string { // The JSON configuration var confData map[string]interface{} + var res bytes.Buffer + err := encjson.Unmarshal(config, &confData) - checkErr(err) + utils.CheckErr(err) // Read the template data, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) tpl, err := template.New("conf").Parse(string(data)) - checkErr(err) + utils.CheckErr(err) - checkErr(tpl.Execute(os.Stdout, confData)) + utils.CheckErr(tpl.Execute(&res, confData)) + return res.String() } -func processHCLTemplate(file string, config []byte) { +func ProcessHCLTemplate(file string, config []byte) string { fct, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1}) - checkDiags(diags) + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) // Retrieve values from JSON var varsVal cty.Value @@ -56,7 +50,7 @@ func processHCLTemplate(file string, config []byte) { if err != nil { panic(err) /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1}) + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) if diags.HasErrors() { panic(diags.Error()) } @@ -66,7 +60,7 @@ func processHCLTemplate(file string, config []byte) { */ } else { varsVal, err = ctyjson.Unmarshal(config, ctyType) - checkErr(err) + utils.CheckErr(err) } ctx := &hcl.EvalContext{ @@ -91,20 +85,5 @@ func processHCLTemplate(file string, config []byte) { panic(diags.Error()) } - fmt.Printf("%s", val.AsString()) -} - -func main() { - // The template to process - templateType := os.Args[1] - templateFile := os.Args[2] - config := []byte(os.Args[3]) - - if templateType == "go" { - processGoTemplate(templateFile, config) - } else if templateType == "hcl" { - processHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) - } + return val.AsString() } diff --git a/pkg/utils/main.go b/pkg/utils/main.go new file mode 100644 index 0000000..8ea276d --- /dev/null +++ b/pkg/utils/main.go @@ -0,0 +1,17 @@ +package utils + +import ( + "github.com/hashicorp/hcl/v2" +) + +func CheckErr(e error) { + if e != nil { + panic(e) + } +} + +func CheckDiags(diag hcl.Diagnostics) { + if diag.HasErrors() { + panic(diag.Error()) + } +} From b5172da6a7322b86381953d9859010c76f6bc736 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Wed, 15 Jun 2022 17:29:45 +0200 Subject: [PATCH 4/9] Complete rewrite of the bootstraper tool --- .gitignore | 2 +- Makefile | 6 +- api/main.go | 10 +- cmd/bootstraper/main.go | 27 ++++ cmd/{templater.go => templater/main.go} | 14 +- data/config/test-services.hcl | 127 +++++++++++++++++ data/config/test-services.json | 127 +++++++++++++++++ data/config/test.json | 26 ---- data/schema/templater.hcl | 0 data/schema/templater.json | 0 data/templates/grafana.ini.pktpl.hcl | 5 + data/templates/hosts.pktpl.hcl | 0 data/templates/loki-local-config.pktpl.hcl | 42 +++--- data/templates/loki-local-config.tpl | 30 ++-- go.mod | 38 +++++ pkg/templater/files.go | 153 +++++++++++++++++++++ pkg/templater/main.go | 145 ++++++++++--------- pkg/templater/packages.go | 74 ++++++++++ pkg/templater/persist.go | 51 +++++++ pkg/templater/repo-apk.go | 102 ++++++++++++++ pkg/templater/repo-deb.go | 43 ++++++ pkg/templater/repo-helm.go | 25 ++++ pkg/templater/repo.go | 17 +++ pkg/templater/services.go | 122 ++++++++++++++++ pkg/templater/system_services.go | 132 ++++++++++++++++++ pkg/templater/system_users.go | 51 +++++++ pkg/utils/main.go | 15 ++ pkg/utils/os.go | 27 ++++ 28 files changed, 1263 insertions(+), 148 deletions(-) create mode 100644 cmd/bootstraper/main.go rename cmd/{templater.go => templater/main.go} (86%) create mode 100644 data/config/test-services.hcl create mode 100644 data/config/test-services.json delete mode 100644 data/config/test.json create mode 100644 data/schema/templater.hcl create mode 100644 data/schema/templater.json create mode 100644 data/templates/grafana.ini.pktpl.hcl create mode 100644 data/templates/hosts.pktpl.hcl create mode 100644 go.mod create mode 100644 pkg/templater/files.go create mode 100644 pkg/templater/packages.go create mode 100644 pkg/templater/persist.go create mode 100644 pkg/templater/repo-apk.go create mode 100644 pkg/templater/repo-deb.go create mode 100644 pkg/templater/repo-helm.go create mode 100644 pkg/templater/repo.go create mode 100644 pkg/templater/services.go create mode 100644 pkg/templater/system_services.go create mode 100644 pkg/templater/system_users.go create mode 100644 pkg/utils/os.go diff --git a/.gitignore b/.gitignore index f4d432a..5bbb8bf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ # Dependency directories (remove the comment below to include it) # vendor/ - +bin/ diff --git a/Makefile b/Makefile index 254cad2..a8b6c47 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux cmd/templater.go + GOOS=linux CGO_ENABLED=0 go build -o bin/templater-linux cmd/templater/main.go + GOOS=linux CGO_ENABLED=0 go build -o bin/bootstraper-linux cmd/bootstraper/main.go upx bin/templater-linux - upx bin/templaster-server + upx bin/bootstraper-linux install: cp bin/templater-linux $(DESTDIR)/bin/templater + cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper uninstall: rm $(DESTDIR)/bin/templater diff --git a/api/main.go b/api/main.go index 7c12c62..d0766f1 100644 --- a/api/main.go +++ b/api/main.go @@ -1,9 +1,6 @@ package api import ( - "net/http" - - "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" "github.com/gin-gonic/gin" ) @@ -13,6 +10,12 @@ type Template struct { Config string } +func Generate(c *gin.Context) { + return +} + +/* + func Generate(c *gin.Context) { var template Template @@ -42,3 +45,4 @@ func Generate(c *gin.Context) { } } +*/ diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go new file mode 100644 index 0000000..68f0732 --- /dev/null +++ b/cmd/bootstraper/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" +) + +func main() { + + var args struct { + Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"` + TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"` + RootDirectory string `arg:"-r,--root-dir,env:ROOT_DIR" help:"Generate files with this root instead of /" default:"/"` + } + + arg.MustParse(&args) + + var hostConfig templater.TemplaterConfig + + err := hostConfig.New(args.Config, args.TemplateDirectory) + if err != nil { + panic(err) + } + if err = hostConfig.ManageServices(args.RootDirectory); err != nil { + panic(err) + } +} diff --git a/cmd/templater.go b/cmd/templater/main.go similarity index 86% rename from cmd/templater.go rename to cmd/templater/main.go index 904353e..5fba371 100644 --- a/cmd/templater.go +++ b/cmd/templater/main.go @@ -55,13 +55,13 @@ func main() { config = []byte(args.Config) } - result := "" - if templateType == "go" { - result = templater.ProcessGoTemplate(templateFile, config) - } else if templateType == "hcl" { - result = templater.ProcessHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) + var file templater.ConfigFile + file.Source = templateFile + file.TemplateType = templateType + + result, err := file.ProcessTemplate(templateFile, config) + if err != nil { + panic(err) } if output == "stdout" { fmt.Printf("%s", result) diff --git a/data/config/test-services.hcl b/data/config/test-services.hcl new file mode 100644 index 0000000..80d5aee --- /dev/null +++ b/data/config/test-services.hcl @@ -0,0 +1,127 @@ +LokiStack = { + Name: "LokiStack" + Global: { + Vars: {} + ConfigFiles: [ + { + destination: /etc/hosts + source: hosts.pktpl.hcl + mode: 600 + owner: root + group: root + } + ] + } + Services: { + Loki: { + ConfigFiles: [ + { + destination: /etc/loki/loki-local-config.yaml + source: loki-local-config.pktpl.hcl + mode: 600 + owner: loki + service: loki + group: grafana + } + ] + Repositories: { + Grafana: { + type: helm + name: grafana + url: https://grafana.github.io/helm-charts + enabled:true + } + AlpineTesting: { + type: apk + name: testing + url: http://mirrors.bfsu.edu.cn/alpine/edge/testing + enabled: true + } + } + Packages: { + loki: { + name: loki + action: install + } + promtail: { + name: loki-promtail + action: install + } + loki-helm: { + type: helm + name: loki + repo: grafana/loki-simple-scalable + } + } + Vars: { + AuthEnabled: false + User: loki + Group: grafana + HTTPPort: 3100 + GRPCPort: 9096 + AlertManagerURL: http://localhost:9093 + StorageRoot: /var/loki + SharedStore: filesystem + ObjectStore: filesystem + LogLevel: error + S3: { + URL: + BucketName: + APIKey: + APISecretKey: + } + } + Daemons: { + Loki: { + name: loki + enabled: true + } + } + Users: { + loki: { + username: loki + group: grafana + home : /srv/loki + shell: /bin/nologin + } + } + } + Grafana: { + ConfigFiles: [ + { + destination: /etc/grafana.ini + source: grafana.ini.pktpl.hcl + mode: 700 + owner: grafana + group: grafana + } + ] + Packages: { + grafana: { + name: grafana + action: install + } + } + Vars: { + AuthEnabled: false + User: toto + Group: grafana + } + Users: { + grafana: { + username: grafana + group: grafana + home: /srv/grafana + shell: /bin/nologin + } + } + Daemons: { + grafana: { + name: grafana + type: auto + enabled: true + } + } + } + } +} \ No newline at end of file diff --git a/data/config/test-services.json b/data/config/test-services.json new file mode 100644 index 0000000..66855d4 --- /dev/null +++ b/data/config/test-services.json @@ -0,0 +1,127 @@ +{ + "Name": "LokiStack", + "Global": { + "Vars": {}, + "ConfigFiles": [ + { + "destination": "/etc/hosts", + "source": "hosts.pktpl.hcl", + "mode": "600", + "owner": "root", + "group": "root" + } + ] + }, + "Services": { + "Loki": { + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mode": "600", + "owner": "loki", + "service": "loki", + "group": "grafana" + } + ], + "Repositories": { + "Grafana": { + "type": "helm", + "name": "grafana", + "url": "https://grafana.github.io/helm-charts", + "enabled":true + }, + "AlpineTesting": { + "type": "apk", + "name": "testing", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing", + "enabled": true + } + }, + "Packages": { + "loki": { + "name": "loki", + "action": "install" + }, + "promtail": { + "name": "loki-promtail", + "action": "install" + }, + "loki-helm": { + "type": "helm", + "name": "loki", + "repo": "grafana/loki-simple-scalable" + } + }, + "Vars": { + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } + }, + "Daemons": { + "Loki": { + "name": "loki", + "enabled": true + } + }, + "Users": { + "loki": { + "username": "loki", + "group": "grafana", + "home" : "/srv/loki", + "shell": "/bin/nologin" + } + } + }, + "Grafana": { + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "source": "grafana.ini.pktpl.hcl", + "mode": "700", + "owner": "grafana", + "group": "grafana" + } + ], + "Packages": { + "grafana": { + "name": "grafana", + "action": "install" + } + }, + "Vars": { + "AuthEnabled": false, + "User": "toto", + "Group": "grafana" + }, + "Users": { + "grafana": { + "username": "grafana", + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin" + } + }, + "Daemons": { + "grafana": { + "name": "grafana", + "type": "auto", + "enabled": true + } + } + } + } +} \ No newline at end of file diff --git a/data/config/test.json b/data/config/test.json deleted file mode 100644 index 14035cb..0000000 --- a/data/config/test.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Name": "loki", - "ConfigFiles": [ - { - "destination": "/etc/loki/loki-local-config.yaml", - "source": "loki-local-config.pktpl.hcl", - "mod": "600" - } - ], - "AuthEnabled": false, - "User": "loki", - "Group": "grafana", - "HTTPPort": "3100", - "GRPCPort": "9096", - "AlertManagerURL": "http://localhost:9093", - "StorageRoot": "/var/loki", - "SharedStore": "filesystem", - "ObjectStore": "filesystem", - "LogLevel": "error", - "S3": { - "URL": "", - "BucketName": "", - "APIKey": "", - "APISecretKey": "" - } -} \ No newline at end of file diff --git a/data/schema/templater.hcl b/data/schema/templater.hcl new file mode 100644 index 0000000..e69de29 diff --git a/data/schema/templater.json b/data/schema/templater.json new file mode 100644 index 0000000..e69de29 diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl new file mode 100644 index 0000000..3ba92f6 --- /dev/null +++ b/data/templates/grafana.ini.pktpl.hcl @@ -0,0 +1,5 @@ +%{ if Vars.AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } \ No newline at end of file diff --git a/data/templates/hosts.pktpl.hcl b/data/templates/hosts.pktpl.hcl new file mode 100644 index 0000000..e69de29 diff --git a/data/templates/loki-local-config.pktpl.hcl b/data/templates/loki-local-config.pktpl.hcl index 9fd10f7..0e64c43 100644 --- a/data/templates/loki-local-config.pktpl.hcl +++ b/data/templates/loki-local-config.pktpl.hcl @@ -1,18 +1,18 @@ -%{ if AuthEnabled ~} +%{ if Vars.AuthEnabled ~} auth_enabled: true %{ else } auth_enabled: false %{ endif } server: - http_listen_port: ${HTTPPort} - grpc_listen_port: ${GRPCPort} - log_level: ${LogLevel} + http_listen_port: ${Vars.HTTPPort} + grpc_listen_port: ${Vars.GRPCPort} + log_level: ${Vars.LogLevel} ingester: wal: enabled: true - dir: ${StorageRoot}/wal + dir: ${Vars.StorageRoot}/wal flush_on_shutdown: true lifecycler: address: 127.0.0.1 @@ -23,7 +23,7 @@ ingester: final_sleep: 0s chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h - chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_target_size: 1048576 # Vars. will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) max_transfer_retries: 0 # Chunk transfers disabled @@ -31,31 +31,31 @@ schema_config: configs: - from: 2020-05-15 store: boltdb-shipper - object_store: ${ObjectStore} + object_store: ${Vars.ObjectStore} schema: v11 index: prefix: index_ period: 24h storage_config: - boltdb_shipper: - active_index_directory: ${StorageRoot}/index - shared_store: ${SharedStore} - cache_location: ${StorageRoot}/cache - cache_ttl: 168h + boltdb_shipper: + active_index_directory: ${Vars.StorageRoot}/index + shared_store: ${Vars.SharedStore} + cache_location: ${Vars.StorageRoot}/cache + cache_ttl: 168h -%{ if ObjectStore == "filesystem" ~} +%{ if Vars.ObjectStore == "filesystem" ~} filesystem: - directory: ${StorageRoot}/chunks + directory: ${Vars.StorageRoot}/chunks %{ else } aws: - s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName} + s3: s3://${Vars.S3.APIKey}:${Vars.S3.APISecretKey}@${Vars.S3.URL}/${Vars.S3.BucketName} s3forcepathstyle: true %{ endif } compactor: - shared_store: ${SharedStore} - working_directory: ${StorageRoot}/compactor + shared_store: ${Vars.SharedStore} + working_directory: ${Vars.StorageRoot}/compactor compaction_interval: 10m limits_config: @@ -73,10 +73,10 @@ ruler: storage: type: local local: - directory: ${StorageRoot}/rules - rule_path: ${StorageRoot}/rules - alertmanager_url: ${AlertManagerURL} + directory: ${Vars.StorageRoot}/rules + rule_path: ${Vars.StorageRoot}/rules + alertmanager_url: ${Vars.AlertManagerURL} ring: kvstore: store: inmemory - enable_api: true \ No newline at end of file + enable_api: true diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl index 07cd8c8..a0de866 100644 --- a/data/templates/loki-local-config.tpl +++ b/data/templates/loki-local-config.tpl @@ -1,18 +1,18 @@ -{{ if .AuthEnabled }} +{{ if .Vars.AuthEnabled }} auth_enabled: true {{ else }} auth_enabled: false {{ end }} server: - http_listen_port: {{ .HTTPPort }} - grpc_listen_port: {{ .GRPCPort }} + http_listen_port: {{ .Vars.HTTPPort }} + grpc_listen_port: {{ .Vars.GRPCPort }} log_level: {{ .LogLevel }} ingester: wal: enabled: true - dir: {{ .StorageRoot }}/wal + dir: {{ .Vars.StorageRoot }}/wal flush_on_shutdown: true lifecycler: address: 127.0.0.1 @@ -31,7 +31,7 @@ schema_config: configs: - from: 2020-05-15 store: boltdb-shipper - object_store: {{ .ObjectStore }} + object_store: {{ .Vars.ObjectStore }} schema: v11 index: prefix: index_ @@ -39,23 +39,23 @@ schema_config: storage_config: boltdb_shipper: - active_index_directory: {{ .StorageRoot }}/index - shared_store: {{ .SharedStore }} - cache_location: {{ .StorageRoot }}/cache + active_index_directory: {{ .Vars.StorageRoot }}/index + shared_store: {{ .Vars.SharedStore }} + cache_location: {{ .Vars.StorageRoot }}/cache cache_ttl: 168h {{ if eq (.ObjectStore) ("filesystem") }} filesystem: - directory: {{ .StorageRoot }}/chunks + directory: {{ .Vars.StorageRoot }}/chunks {{ else }} aws: - s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} + s3: s3://{{ .Vars.S3.APIKey }}:{{ .Vars.S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} s3forcepathstyle: true {{ end }} compactor: - shared_store: {{ .SharedStore }} - working_directory: {{ .StorageRoot }}/compactor + shared_store: {{ .Vars.SharedStore }} + working_directory: {{ .Vars.StorageRoot }}/compactor compaction_interval: 10m limits_config: @@ -73,9 +73,9 @@ ruler: storage: type: local local: - directory: {{ .StorageRoot }}/rules - rule_path: {{ .StorageRoot }}/rules - alertmanager_url: {{ .AlertManagerURL }} + directory: {{ .Vars.StorageRoot }}/rules + rule_path: {{ .Vars.StorageRoot }}/rules + alertmanager_url: {{ .Vars.AlertManagerURL }} ring: kvstore: store: inmemory diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..71790bf --- /dev/null +++ b/go.mod @@ -0,0 +1,38 @@ +module forge.cadoles.com/pcaseiro/templatefile + +go 1.18 + +require ( + github.com/alexflint/go-arg v1.4.3 + github.com/gin-gonic/gin v1.8.1 + github.com/hashicorp/hcl/v2 v2.11.1 + github.com/imdario/mergo v0.3.13 + github.com/zclconf/go-cty v1.10.0 + gopkg.in/ini.v1 v1.66.6 +) + +require ( + github.com/agext/levenshtein v1.2.1 // indirect + github.com/alexflint/go-scalar v1.1.0 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/goccy/go-json v0.9.7 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/ugorji/go/codec v1.2.7 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + golang.org/x/text v0.3.6 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/pkg/templater/files.go b/pkg/templater/files.go new file mode 100644 index 0000000..c5428ab --- /dev/null +++ b/pkg/templater/files.go @@ -0,0 +1,153 @@ +package templater + +import ( + "log" + "path/filepath" + "strconv" + + "bytes" + encjson "encoding/json" + "fmt" + "os" + "text/template" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type ConfigFile struct { + Destination string `form:"destination" json:"destination"` // Where do we write the configuration file + Source string `form:"source" json:"source"` // The template file short name + TemplateType string `json:"type"` // The template file type (hcl or gotemplate) + Mode string `form:"mod" json:"mode"` // The configuration file final permissions (mode) + Owner string `json:"owner"` // The configuration file owner + Service string `json:"service"` // Service to restart after configuration generation + Group string `json:"group"` // The configuration file group owner +} + +// Generate the configuration file from the template (hcl or json) +func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error { + var template string + dest := filepath.Join(root, cf.Destination) + source := filepath.Join(templateDir, cf.Source) + intMod, err := strconv.ParseInt(cf.Mode, 8, 64) + if err != nil { + return (err) + } + + template, err = cf.ProcessTemplate(source, values) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + dirname := filepath.Dir(dest) + err = os.MkdirAll(dirname, os.FileMode(int(0700))) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + err = os.WriteFile(dest, []byte(template), os.FileMode(intMod)) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + log.Printf("\tFile %s generated\n", dest) + return nil +} + +// Process the template with the provided values +func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) { + var result string + var err error + + if cf.TemplateType == "hcl" { + // The template is an hcl template so we call processHCLTemplate + result, err = cf.processHCLTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process HCL template failed with error: %v", err) + } + } else if cf.TemplateType == "go" { + // The template is a go template so we call processGoTemplate + result, err = cf.processGoTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process GO template failed with error: %v", err) + } + } + return result, nil +} + +// The actual template processing for Go templates +func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) { + + // The JSON configuration + var confData map[string]interface{} + var res bytes.Buffer + + err := encjson.Unmarshal(configValues, &confData) + utils.CheckErr(err) + + // Read the template + data, err := os.ReadFile(file) + utils.CheckErr(err) + + tpl, err := template.New("conf").Parse(string(data)) + utils.CheckErr(err) + + utils.CheckErr(tpl.Execute(&res, confData["Config"])) + + return res.String(), nil +} + +// The actual template processing for HCL templates +func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) { + + fct, err := os.ReadFile(file) + utils.CheckErr(err) + + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) + + // Retrieve values from JSON + var varsVal cty.Value + ctyType, err := ctyjson.ImpliedType(config) + if err != nil { + return "", err + /* Maybe one day + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) + if diags.HasErrors() { + panic(diags.Error()) + } + varsVal, diags = cexpr.Value(&hcl.EvalContext{}) + fmt.Println(cexpr.Variables()) + checkDiags(diags) + */ + } else { + varsVal, err = ctyjson.Unmarshal(config, ctyType) + utils.CheckErr(err) + } + + ctx := &hcl.EvalContext{ + Variables: varsVal.AsValueMap(), + } + + for n := range ctx.Variables { + if !hclsyntax.ValidIdentifier(n) { + return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n) + } + } + + for _, traversal := range expr.Variables() { + root := traversal.RootName() + if _, ok := ctx.Variables[root]; !ok { + return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()) + } + } + + val, diags := expr.Value(ctx) + if diags.HasErrors() { + return "", diags + } + + return val.AsString(), nil +} diff --git a/pkg/templater/main.go b/pkg/templater/main.go index f1fba18..8a62ee3 100644 --- a/pkg/templater/main.go +++ b/pkg/templater/main.go @@ -1,89 +1,88 @@ package templater import ( - "bytes" - encjson "encoding/json" "fmt" + "io/ioutil" + "log" "os" - "text/template" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - ctyjson "github.com/zclconf/go-cty/cty/json" - - "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" + "github.com/imdario/mergo" ) -func ProcessGoTemplate(file string, config []byte) string { +var CacheFilePath = "/var/cache/templater.db" - // The JSON configuration - var confData map[string]interface{} - var res bytes.Buffer - - err := encjson.Unmarshal(config, &confData) - utils.CheckErr(err) - - // Read the template - data, err := os.ReadFile(file) - utils.CheckErr(err) - - tpl, err := template.New("conf").Parse(string(data)) - utils.CheckErr(err) - - utils.CheckErr(tpl.Execute(&res, confData)) - - return res.String() +type TemplaterConfig struct { + Name string `json:"Name"` + TemplateDirectory string `json:"TemplateDirectory"` + Services map[string]Service `json:"Services"` + GlobalService Service `json:"Global"` } -func ProcessHCLTemplate(file string, config []byte) string { - - fct, err := os.ReadFile(file) - utils.CheckErr(err) - - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) - utils.CheckDiags(diags) - - // Retrieve values from JSON - var varsVal cty.Value - ctyType, err := ctyjson.ImpliedType(config) +func (tc *TemplaterConfig) loadCache() error { + // Load globals from cache + var cache Service + err := Load(CacheFilePath, &cache) if err != nil { - panic(err) - /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) - if diags.HasErrors() { - panic(diags.Error()) - } - varsVal, diags = cexpr.Value(&hcl.EvalContext{}) - fmt.Println(cexpr.Variables()) - checkDiags(diags) - */ - } else { - varsVal, err = ctyjson.Unmarshal(config, ctyType) - utils.CheckErr(err) + fmt.Printf("Warning: No globals to load\n") } - ctx := &hcl.EvalContext{ - Variables: varsVal.AsValueMap(), + err = mergo.Merge(&tc.GlobalService, cache) + if err != nil { + return err } - - for n := range ctx.Variables { - if !hclsyntax.ValidIdentifier(n) { - panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)) - } - } - - for _, traversal := range expr.Variables() { - root := traversal.RootName() - if _, ok := ctx.Variables[root]; !ok { - panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())) - } - } - - val, diags := expr.Value(ctx) - if diags.HasErrors() { - panic(diags.Error()) - } - - return val.AsString() + return nil +} + +func (tc *TemplaterConfig) New(confpath string, templateDir string) error { + // Load stored globals if needed + lerr := tc.loadCache() + if lerr != nil { + return lerr + } + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(confpath) + if err != nil { + return err + } + + if fileInfo.IsDir() { + // The conf path is a directory we load all the files and merge data + files, err := ioutil.ReadDir(confpath) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + for _, file := range files { + fname := fmt.Sprintf("%s/%s", confpath, file.Name()) + var ntc TemplaterConfig + err := Load(fname, &ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + err = mergo.Merge(tc, ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + } + } else { + // The conf path is a file we only load this file (of course) + err = Load(confpath, tc) + if err != nil { + return fmt.Errorf("Confiuration read failed with error: %v", err) + } + } + + tc.TemplateDirectory = templateDir + + return nil +} + +func (tc *TemplaterConfig) ManageServices(rootDir string) error { + for name, svr := range tc.Services { + log.Printf("*** Working on service %s", name) + if err := svr.Manage(tc.TemplateDirectory, rootDir); err != nil { + return err + } + log.Printf("*** Service %s processed", name) + } + return nil } diff --git a/pkg/templater/packages.go b/pkg/templater/packages.go new file mode 100644 index 0000000..98dca8d --- /dev/null +++ b/pkg/templater/packages.go @@ -0,0 +1,74 @@ +package templater + +import ( + "fmt" + "log" + "runtime" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemPackage struct { + Name string `json:"name"` + Type string `json:"type"` + Action string `json:"action"` + OS string `json:"os"` + Distribution string `json:"distribution"` +} + +func (p *SystemPackage) SetDistribution() error { + OSConfig, err := utils.ReadOSRelease() + if err != nil { + return err + } + + p.Distribution = OSConfig["ID_LIKE"] + return nil +} +func (p *SystemPackage) SetOS() error { + p.OS = runtime.GOOS + return nil +} + +func (p *SystemPackage) Manage() error { + var pkErr error + var stdErr []byte + + if p.OS == "" { + if err := p.SetOS(); err != nil { + return err + } + } + + if p.Distribution == "" { + if err := p.SetDistribution(); err != nil { + return err + } + } + + log.Printf("\tInstalling %s package\n", p.Name) + switch os := p.Distribution; os { + case "debian", "ubuntu": + _, stdErr, pkErr = utils.RunSystemCommand("apt", "install", "-y", p.Name) + case "alpine": + _, stdErr, pkErr = utils.RunSystemCommand("apk", "add", p.Name) + case "redhat": + _, stdErr, pkErr = utils.RunSystemCommand("yum", "install", "-y", p.Name) + case "arch": + _, stdErr, pkErr = utils.RunSystemCommand("pacman", "-Suy", p.Name) + default: + pkErr = fmt.Errorf("Unsupported OS %s [%s]", p.OS, stdErr) + } + + if pkErr != nil { + var msg string + if len(stdErr) != 0 { + msg = string(stdErr) + } else { + msg = pkErr.Error() + } + return fmt.Errorf("Package %s, os %s, failed with error: %v", p.Name, p.OS, msg) + } + + return nil +} diff --git a/pkg/templater/persist.go b/pkg/templater/persist.go new file mode 100644 index 0000000..edc2155 --- /dev/null +++ b/pkg/templater/persist.go @@ -0,0 +1,51 @@ +package templater + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "sync" +) + +var lock sync.Mutex + +func marshal(v interface{}) (io.Reader, error) { + b, err := json.MarshalIndent(v, "", "\t") + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +func unmarshal(r io.Reader, v interface{}) error { + return json.NewDecoder(r).Decode(v) +} + +func Save(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("Saving Templater configuration failed with error : %v", err) + } + defer f.Close() + r, err := marshal(v) + if err != nil { + return err + } + _, err = io.Copy(f, r) + return err +} + +func Load(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return unmarshal(f, v) +} diff --git a/pkg/templater/repo-apk.go b/pkg/templater/repo-apk.go new file mode 100644 index 0000000..c8f88a6 --- /dev/null +++ b/pkg/templater/repo-apk.go @@ -0,0 +1,102 @@ +package templater + +import ( + "bufio" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +var APKConfigFile = "/etc/apk/repositories" + +type APKRepository struct { + Repository +} + +func (hr *APKRepository) urlIsPresent() (bool, error) { + f, err := os.Open(APKConfigFile) + if err != nil { + return false, err + } + defer f.Close() + + // Splits on newlines by default. + scanner := bufio.NewScanner(f) + + line := 1 + for scanner.Scan() { + if strings.Contains(scanner.Text(), hr.URL) { + log.Printf("\tRepository %s already present\n", hr.Name) + return true, nil + } + line++ + } + + if err := scanner.Err(); err != nil { + return false, err + } + return false, nil + +} + +func (hr *APKRepository) Add() error { + + URLIsPresent, err := hr.urlIsPresent() + if err != nil { + return err + } + + if URLIsPresent { + return nil + } else { + data := fmt.Sprintf("%s\n", hr.URL) + file, err := os.OpenFile(APKConfigFile, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + if _, err := file.WriteString(data); err != nil { + return err + } else { + log.Printf("Repository %s added\n", hr.Name) + } + return nil + } + +} + +func (hr *APKRepository) Update() error { + if _, stdErr, err := utils.RunSystemCommand("apk", "update"); err != nil { + return fmt.Errorf("%s [%s]", stdErr, err) + } + return nil +} + +// FIXME +func (hr *APKRepository) Delete() error { + fileBytes, err := ioutil.ReadFile("/etc/apk/repositories") + if err != nil { + return err + } + lines := strings.Split(string(fileBytes), "\n") + for _, line := range lines { + fmt.Printf("DEBUG TODO %s", line) + } + return nil +} + +func (hr *APKRepository) Manage() error { + if hr.Enabled { + if err := hr.Add(); err != nil { + return err + } + log.Println("\tUpdating apk repositories") + return hr.Update() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo-deb.go b/pkg/templater/repo-deb.go new file mode 100644 index 0000000..1c6f6f9 --- /dev/null +++ b/pkg/templater/repo-deb.go @@ -0,0 +1,43 @@ +package templater + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type DebRepository struct { + Repository +} + +func (hr *DebRepository) Add() error { + //deb http://fr.archive.ubuntu.com/ubuntu/ focal main restricted + + data := fmt.Sprintf("deb %s", hr.URL) + if err := os.WriteFile("/etc/apt/source.list.d", []byte(data), 0600); err != nil { + return err + } + + return nil +} + +func (hr *DebRepository) Update() error { + if _, stdErr, err := utils.RunSystemCommand("apt", "update", "-y"); err != nil { + return fmt.Errorf("%s [%s]", stdErr, err) + } + + return nil +} + +func (hr *DebRepository) Delete() error { + //TODO + return nil +} +func (hr *DebRepository) Manage() error { + if hr.Enabled { + return hr.Add() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo-helm.go b/pkg/templater/repo-helm.go new file mode 100644 index 0000000..71f1803 --- /dev/null +++ b/pkg/templater/repo-helm.go @@ -0,0 +1,25 @@ +package templater + +type HelmRepository struct { + Repository +} + +func (hr *HelmRepository) Add() error { + return nil +} + +func (hr *HelmRepository) Update() error { + return nil +} + +func (hr *HelmRepository) Delete() error { + return nil +} + +func (hr *HelmRepository) Manage() error { + if hr.Enabled { + return hr.Add() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo.go b/pkg/templater/repo.go new file mode 100644 index 0000000..8c143d0 --- /dev/null +++ b/pkg/templater/repo.go @@ -0,0 +1,17 @@ +package templater + +type PackageRepository interface { + Manage() error + Update() error + Add() error + Delete() error +} + +type Repository struct { + Actions PackageRepository + + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + Enabled bool `json:"enabled"` +} diff --git a/pkg/templater/services.go b/pkg/templater/services.go new file mode 100644 index 0000000..b95779d --- /dev/null +++ b/pkg/templater/services.go @@ -0,0 +1,122 @@ +package templater + +import ( + "encoding/json" + "fmt" + "log" + "path/filepath" +) + +type Service struct { + ConfigFiles []ConfigFile `json:"ConfigFiles"` + Vars map[string]interface{} `json:"Vars"` + Daemons map[string]SystemService `json:"Daemons"` + Users map[string]SystemUser `json:"Users"` + Repos map[string]Repository `json:"Repositories"` + Packages map[string]SystemPackage `json:"Packages"` +} + +func (s *Service) manageRepos(repos map[string]Repository) error { + for _, repo := range s.Repos { + if repo.Type == "helm" { + rp := HelmRepository{repo} + if err := rp.Manage(); err != nil { + return err + } + } + if repo.Type == "apk" { + rp := APKRepository{repo} + if err := rp.Manage(); err != nil { + return err + } + } + if repo.Type == "deb" { + rp := DebRepository{} + if err := rp.Manage(); err != nil { + return err + } + } + + } + return nil +} + +func (s *Service) Manage(templateDir string, rootDir string) error { + // Manage packages repositories + log.Print(" Managing package repositories") + err := s.manageRepos(s.Repos) + if err != nil { + return err + } + + // Create system users + log.Print(" Managing system users") + for _, user := range s.Users { + err := user.Manage() + if err != nil { + return err + } + } + + // Manage system packages + log.Print(" Installing packages") + for _, pack := range s.Packages { + err := pack.Manage() + if err != nil { + return err + } + log.Printf("\tPackage %s installed\n", pack.Name) + } + + log.Print(" Generating configuration files\n") + err = processConfigFiles(s, templateDir, rootDir) + if err != nil { + return fmt.Errorf("ProcessingTemplatesFailed with error: %v", err) + } + + log.Print(" Managing services:\n") + for _, daemon := range s.Daemons { + err = daemon.Manage() + if err != nil { + return fmt.Errorf("Error managing service daemons: %v", err) + } + } + return nil +} + +func processConfigFiles(s *Service, templateDir string, rootDir string) error { + values, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("Error unmarshaling values on template process; %v", err) + } + + var servicesToRestart []string + for _, tpl := range s.ConfigFiles { + fileExt := filepath.Ext(tpl.Source) + if fileExt == ".hcl" { + tpl.TemplateType = "hcl" + } else if fileExt == ".tpl" { + tpl.TemplateType = "go" + } else { + return fmt.Errorf("Unsupported file type %s, templates extensions have to be '.hcl' or '.tpl'", fileExt) + } + if err := tpl.Generate(rootDir, templateDir, values); err != nil { + return fmt.Errorf("Template %s generation failed with error %v", tpl.Source, err) + } + + if len(tpl.Service) != 0 { + servicesToRestart = append(servicesToRestart, tpl.Service) + } + } + + for _, srv := range servicesToRestart { + sv := SystemService{ + Name: srv, + Enabled: true, + Type: "", + ToStart: true, + } + return sv.Restart() + } + return nil +} diff --git a/pkg/templater/system_services.go b/pkg/templater/system_services.go new file mode 100644 index 0000000..582cb2c --- /dev/null +++ b/pkg/templater/system_services.go @@ -0,0 +1,132 @@ +package templater + +import ( + "fmt" + "log" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemService struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Type string `json:"type"` + ToStart bool `json:"start"` +} + +func (sys *SystemService) SetType() { + systemdRunDirectory := "/run/systemd/system" + openRcBinaryFile := "/sbin/openrc" + + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(systemdRunDirectory) + if err == nil { + if fileInfo.IsDir() { + sys.Type = "systemd" + } + } + + fileInfo, err = os.Stat(openRcBinaryFile) + if err == nil { + if fileInfo.IsDir() { + return + } + sys.Type = "openrc" + } +} + +func (sys *SystemService) Action() error { + if sys.ToStart { + return sys.Start() + } + return nil +} + +func (sys *SystemService) Manage() error { + // By default if the property sys.ToStart is empty + if sys.Type == "" || sys.Type == "auto" { + sys.SetType() + } + if sys.Enabled { + err := sys.Enable() + if err != nil { + return err + } + if err = sys.Action(); err != nil { + return err + } + + } else { + log.Printf("\nNothing to do for daemon %s\n", sys.Name) + } + return nil +} + +func (sys *SystemService) Start() error { + log.Printf("\tStarting system service : %s\n", sys.Name) + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "start", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Start error:\n - %s", sys.Name, stdErr) + } + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("service", sys.Name, "start") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} + +func (sys *SystemService) Stop() error { + log.Printf("\tStopping system service : %s\n", sys.Name) + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "stop", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Stop error:\n - %s", sys.Name, stdErr) + } + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("service", sys.Name, "stop") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} + +func (sys *SystemService) Restart() error { + if sys.Type == "" || sys.Type == "auto" { + sys.SetType() + } + if err := sys.Stop(); err != nil { + return err + } + if err := sys.Start(); err != nil { + return err + } + return nil +} + +func (sys *SystemService) Enable() error { + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "enable", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + log.Printf("\tSystemd service %s enabled", sys.Name) + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("rc-update", "add", sys.Name, "default") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + log.Printf("\tOpenRC service %s enabled", sys.Name) + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} diff --git a/pkg/templater/system_users.go b/pkg/templater/system_users.go new file mode 100644 index 0000000..bd870a9 --- /dev/null +++ b/pkg/templater/system_users.go @@ -0,0 +1,51 @@ +package templater + +import ( + "log" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemUser struct { + UserName string `json:"username"` + Group string `json:"group"` + Home string `json:"home"` + Shell string `json:"shell"` +} + +func (su *SystemUser) exists() (bool, error) { + _, _, err := utils.RunSystemCommand("getent", "passwd", su.UserName) + if err != nil { + return false, err + } + return true, nil +} + +func (su *SystemUser) Manage() error { + exist, _ := su.exists() + if exist { + log.Printf("\tUser %s already exists", su.UserName) + return nil + } + return su.Create() +} + +func (su *SystemUser) Create() error { + _, _, err := utils.RunSystemCommand("useradd", "-b", su.Home, "-m", "-G", su.Group, su.UserName) + if err != nil { + return err + } + return nil +} + +func (su *SystemUser) Delete() error { + _, _, err := utils.RunSystemCommand("userdel", su.UserName) + if err != nil { + return err + } + return nil +} + +func (su *SystemUser) Update() error { + return nil +} diff --git a/pkg/utils/main.go b/pkg/utils/main.go index 8ea276d..2c92b51 100644 --- a/pkg/utils/main.go +++ b/pkg/utils/main.go @@ -1,6 +1,9 @@ package utils import ( + "bytes" + "os/exec" + "github.com/hashicorp/hcl/v2" ) @@ -15,3 +18,15 @@ func CheckDiags(diag hcl.Diagnostics) { panic(diag.Error()) } } + +// Execute a system command ... +func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) { + var stdOut bytes.Buffer + var stdErr bytes.Buffer + + cmd := exec.Command(name, arg...) + cmd.Stderr = &stdErr + cmd.Stdout = &stdOut + err := cmd.Run() + return stdOut.Bytes(), stdErr.Bytes(), err +} diff --git a/pkg/utils/os.go b/pkg/utils/os.go new file mode 100644 index 0000000..8fe4d64 --- /dev/null +++ b/pkg/utils/os.go @@ -0,0 +1,27 @@ +package utils + +import ( + "fmt" + + ini "gopkg.in/ini.v1" +) + +var osReleaseFile = "/etc/os-release" + +func ReadOSRelease() (map[string]string, error) { + cfg, err := ini.Load(osReleaseFile) + if err != nil { + return nil, fmt.Errorf("Fail to read file: %v ", err) + } + + ConfigParams := make(map[string]string) + ConfigParams["ID"] = cfg.Section("").Key("ID").String() + idLike := cfg.Section("").Key("ID_LIKE").String() + if idLike != "" { + ConfigParams["ID_LIKE"] = idLike + } else { + ConfigParams["ID_LIKE"] = ConfigParams["ID"] + } + + return ConfigParams, nil +} From 9bb4e93d65fc00dc725a9d034f1e47057c7d866a Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Fri, 1 Jul 2022 09:58:22 +0200 Subject: [PATCH 5/9] fix(bootstraper): adding system group management --- pkg/templater/system_group.go | 49 +++++++++++++++++++++++++++++++++++ pkg/templater/system_users.go | 10 ++++++- 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 pkg/templater/system_group.go diff --git a/pkg/templater/system_group.go b/pkg/templater/system_group.go new file mode 100644 index 0000000..deffa8a --- /dev/null +++ b/pkg/templater/system_group.go @@ -0,0 +1,49 @@ +package templater + +import ( + "fmt" + "log" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemGroup struct { + GroupName string `json:"groupname"` +} + +func (sg *SystemGroup) exists() (bool, error) { + _, _, err := utils.RunSystemCommand("getent", "group", sg.GroupName) + if err != nil { + return false, err + } + return true, nil +} + +func (sg *SystemGroup) Manage() error { + exist, _ := sg.exists() + if exist { + log.Printf("\tGroup %s already exists", sg.GroupName) + return nil + } + return sg.Create() +} + +func (sg *SystemGroup) Create() error { + _, stdErr, err := utils.RunSystemCommand("groupadd", "-r", sg.GroupName) + if err != nil { + return fmt.Errorf("Group %s creation failed with error: %s %v", sg.GroupName, stdErr, err) + } + return nil +} + +func (sg *SystemGroup) Delete() error { + _, _, err := utils.RunSystemCommand("userdel", sg.GroupName) + if err != nil { + return err + } + return nil +} + +func (sg *SystemGroup) Update() error { + return nil +} diff --git a/pkg/templater/system_users.go b/pkg/templater/system_users.go index bd870a9..4b66925 100644 --- a/pkg/templater/system_users.go +++ b/pkg/templater/system_users.go @@ -1,6 +1,7 @@ package templater import ( + "fmt" "log" "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" @@ -31,10 +32,17 @@ func (su *SystemUser) Manage() error { } func (su *SystemUser) Create() error { - _, _, err := utils.RunSystemCommand("useradd", "-b", su.Home, "-m", "-G", su.Group, su.UserName) + // Manage System Group + grp := SystemGroup{GroupName: su.Group} + err := grp.Manage() if err != nil { return err } + + _, stdErr, err := utils.RunSystemCommand("useradd", "-b", su.Home, "-m", "-N", "-g", su.Group, su.UserName) + if err != nil { + return fmt.Errorf("User %s creation failed with error: %s %v", su.UserName, stdErr, err) + } return nil } From e9f8ff2506b2d15cf51cc2de08a23de1bc1c7bff Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Fri, 1 Jul 2022 14:56:41 +0200 Subject: [PATCH 6/9] fix(bootstraper): adding global variables support Now we can declare "global" variables, in the "Globals" section the the configuration, this variables will be merged into the "Service" variables and used in the templates. This how we share variables between services. --- cmd/bootstraper/main.go | 4 +- data/config/loki-stack.json | 204 ++++ data/templates/alertmanager.yml.pktpl.hcl | 16 + data/templates/grafana.ini.pktpl.hcl | 1160 +++++++++++++++++++- data/templates/loki-local-config.pktpl.hcl | 4 +- data/templates/loki-local-config.tpl | 30 +- data/templates/prometheus.yml.pktpl.hcl | 34 + pkg/templater/main.go | 25 +- 8 files changed, 1447 insertions(+), 30 deletions(-) create mode 100644 data/config/loki-stack.json create mode 100644 data/templates/alertmanager.yml.pktpl.hcl create mode 100644 data/templates/prometheus.yml.pktpl.hcl diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go index 68f0732..9701963 100644 --- a/cmd/bootstraper/main.go +++ b/cmd/bootstraper/main.go @@ -17,11 +17,11 @@ func main() { var hostConfig templater.TemplaterConfig - err := hostConfig.New(args.Config, args.TemplateDirectory) + err := hostConfig.New(args.Config, args.TemplateDirectory, args.RootDirectory) if err != nil { panic(err) } - if err = hostConfig.ManageServices(args.RootDirectory); err != nil { + if err = hostConfig.ManageServices(); err != nil { panic(err) } } diff --git a/data/config/loki-stack.json b/data/config/loki-stack.json new file mode 100644 index 0000000..db657ea --- /dev/null +++ b/data/config/loki-stack.json @@ -0,0 +1,204 @@ +{ + "Globals": { + "Vars": { + "PrometheusPort": "9090" + } + }, + "Name": "loki-stack", + "Services": { + "Alertmanager": { + "ConfigFiles": [ + { + "destination": "/etc/alertmanager/alertmanager.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "alertmanager.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "alertmanager", + "type": "auto" + } + }, + "Packages": { + "alertmanager": { + "action": "install", + "name": "alertmanager" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": {} + }, + "Grafana": { + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "group": "grafana", + "mode": "600", + "owner": "grafana", + "source": "grafana.ini.pktpl.hcl" + } + ], + "Daemons": { + "grafana": { + "enabled": true, + "name": "grafana", + "type": "auto" + } + }, + "Packages": { + "grafana": { + "action": "install", + "name": "grafana" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + } + }, + "Users": { + "grafana": { + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin", + "username": "grafana" + } + }, + "Vars": { + "AppMode": "production", + "DomainName": "www.grafana.local", + "HTTPPort": "80", + "HostName": "grafana.local", + "UserName": "grafana" + } + }, + "Loki": { + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "group": "grafana", + "mode": "600", + "owner": "loki", + "service": "loki", + "source": "loki-local-config.pktpl.hcl" + } + ], + "Daemons": { + "Loki": { + "enabled": true, + "name": "loki" + } + }, + "Packages": { + "loki": { + "action": "install", + "name": "loki" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "promtail": { + "action": "install", + "name": "loki-promtail" + } + }, + "Repositories": { + "AlpineTesting": { + "enabled": true, + "name": "testing", + "type": "apk", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing" + } + }, + "Users": { + "loki": { + "group": "grafana", + "home": "/srv/loki", + "shell": "/bin/nologin", + "username": "loki" + } + }, + "Vars": { + "AlertManagerURL": "http://localhost:9092", + "AuthEnabled": false, + "GRPCPort": "9095", + "Group": "grafana", + "HTTPPort": "3099", + "LogLevel": "error", + "ObjectStore": "filesystem", + "S2": { + "APIKey": "", + "APISecretKey": "", + "BucketName": "", + "URL": "" + }, + "SharedStore": "filesystem", + "StorageRoot": "/var/loki", + "User": "loki" + } + }, + "Prometheus": { + "ConfigFiles": [ + { + "destination": "/etc/prometheus/prometheus.yml", + "group": "prometheus", + "mode": "600", + "owner": "prometheus", + "source": "prometheus.yml.pktpl.hcl" + } + ], + "Daemons": { + "prometheus": { + "enabled": true, + "name": "prometheus", + "type": "auto" + } + }, + "Packages": { + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "prometheus": { + "action": "install", + "name": "prometheus" + } + }, + "Users": { + "prometheus": { + "group": "prometheus", + "home": "/var/lib/prometheus", + "shell": "/sbin/nologin", + "username": "prometheus" + } + }, + "Vars": { + "Scrapers": [ + { + "MetricsPath": "/metrics", + "Name": "Prometheus", + "Scheme": "http", + "Targets": [ + "localhost:9001" + ] + } + ] + } + } + } +} diff --git a/data/templates/alertmanager.yml.pktpl.hcl b/data/templates/alertmanager.yml.pktpl.hcl new file mode 100644 index 0000000..7bd5e0c --- /dev/null +++ b/data/templates/alertmanager.yml.pktpl.hcl @@ -0,0 +1,16 @@ +route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 1h + receiver: 'web.hook' +receivers: + - name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:5001/' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl index 3ba92f6..8b42f6d 100644 --- a/data/templates/grafana.ini.pktpl.hcl +++ b/data/templates/grafana.ini.pktpl.hcl @@ -1,5 +1,1155 @@ -%{ if Vars.AuthEnabled ~} -auth_enabled: true -%{ else } -auth_enabled: false -%{ endif } \ No newline at end of file +#####################.Configuration Example ##################### +# +# Everything has defaults so you only need to uncomment things you want to +# change + +# possible values : production, development +app_mode = ${Vars.AppMode} + +# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty +instance_name = ${Vars.HostName} + +# force migration will run migrations that might cause dataloss +;force_migration = false + +#################################### Paths #################################### +[paths] +# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) +data = /var/lib/grafana + +# Temporary files in `data` directory older than given duration will be removed +temp_data_lifetime = 24h + +# Directory where grafana can store logs +logs = /var/log/grafana + +# Directory where grafana will automatically scan and look for plugins +plugins = /var/lib/grafana/plugins + +# folder that contains provisioning config files that grafana will apply on startup and while running. +provisioning = conf/provisioning + +#################################### Server #################################### +[server] +# Protocol (http, https, h2, socket) +;protocol = http + +# The ip address to bind to, empty will bind to all interfaces +;http_addr = + +# The http port to use +http_port = ${Vars.HTTPPort} + +# The public facing domain name used to access grafana from a browser +domain = ${Vars.DomainName} + +# Redirect to correct domain if host header does not match domain +# Prevents DNS rebinding attacks +;enforce_domain = false + +# The full public facing url you use in browser, used for redirects and emails +# If you use reverse proxy and sub path specify full url (with sub path) +;root_url = %(protocol)s://%(domain)s:%(http_port)s/ + +# Serve.from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. +;serve_from_sub_path = false + +# Log web requests +;router_logging = false + +# the path relative working path +;static_root_path = public + +# enable gzip +;enable_gzip = false + +# https certs & key file +;cert_file = +;cert_key = + +# Unix socket path +;socket = + +# CDN Url +;cdn_url = + +# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. +# `0` means there is no timeout for reading the request. +;read_timeout = 0 + +#################################### Database #################################### +[database] +# You can configure the database connection by specifying type, host, name, user and password +# as separate properties or as on string using the url properties. + +# Either "mysql", "postgres" or "sqlite3", it's your choice +type = sqlite3 +host = 127.0.0.1:3306 +name = grafana +user = ${Vars.UserName} +# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" +;password = + +# Use either URL or the previous fields to configure the database +# Example: mysql://user:secret@host:port/database +;url = + +# For "postgres" only, either "disable", "require" or "verify-full" +;ssl_mode = disable + +# Database drivers may support different transaction isolation levels. +# Currently, only "mysql" driver supports isolation levels. +# If the value is empty - driver's default isolation level is applied. +# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". +;isolation_level = + +;ca_cert_path = +;client_key_path = +;client_cert_path = +;server_cert_name = + +# For "sqlite3" only, path relative to data_path setting +path = grafana.db + +# Max idle conn setting default is 2 +;max_idle_conn = 2 + +# Max conn setting default is 0 (mean not set) +;max_open_conn = + +# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) +;conn_max_lifetime = 14400 + +# Set to true to log the sql calls and execution times. +;log_queries = + +# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) +;cache_mode = private + +# For "mysql" only if lockingMigration feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. +;locking_attempt_timeout_sec = 0 + +################################### Data sources ######################### +[datasources] +# Upper limit of data sources that.will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. +;datasource_limit = 5000 + +#################################### Cache server ############################# +[remote_cache] +# Either "redis", "memcached" or "database" default is "database" +;type = database + +# cache connectionstring options +# database: will use.primary database. +# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. +# memcache: 127.0.0.1:11211 +;connstr = + +#################################### Data proxy ########################### +[dataproxy] + +# This enables data proxy logging, default is false +;logging = false + +# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. +# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. +;timeout = 30 + +# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. +;dialTimeout = 10 + +# How many seconds the data proxy waits before sending a keepalive probe request. +;keep_alive_seconds = 30 + +# How many seconds the data proxy waits for a successful TLS Handshake before timing out. +;tls_handshake_timeout_seconds = 10 + +# How many seconds the data proxy will wait for a server's first response headers after +# fully writing the request headers if the request has an "Expect: 100-continue" +# header. A value of 0 will result in the body being sent immediately, without +# waiting for the server to approve. +;expect_continue_timeout_seconds = 1 + +# Optionally limits the total number of connections per host, including connections in the dialing, +# active, and idle states. On limit violation, dials will block. +# A value of zero (0) means no limit. +;max_conns_per_host = 0 + +# The maximum number of idle connections that.will keep alive. +;max_idle_connections = 100 + +# How many seconds the data proxy keeps an idle connection open before timing out. +;idle_conn_timeout_seconds = 90 + +# If enabled and user is not anonymous, data proxy will add X.User header with username into the request, default is false. +;send_user_header = false + +# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. +;response_limit = 0 + +# Limits the number of rows that.will process from SQL data sources. +;row_limit = 1000000 + +#################################### Analytics #################################### +[analytics] +# Server reporting, sends usage counters to stats.grafana.org every 24 hours. +# No ip addresses are being tracked, only simple counters to track +# running instances, dashboard and error counts. It is very helpful to us. +# Change this option to false to disable reporting. +;reporting_enabled = true + +# The name of the distributor of the.instance. Ex hosted-grafana, grafana-labs +;reporting_distributor = grafana-labs + +# Set to false to disable all checks to https://grafana.com +# for new versions of grafana. The check is used +# in some UI views to notify that a grafana update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. +;check_for_updates = true + +# Set to false to disable all checks to https://grafana.com +# for new versions of plugins. The check is used +# in some UI views to notify that a plugin update exists. +# This option does not cause any auto updates, nor send any information +# only a GET request to https://grafana.com to get the latest versions. +;check_for_plugin_updates = true + +# Google Analytics universal tracking code, only enabled if you specify an id here +;google_analytics_ua_id = + +# Google Tag Manager ID, only enabled if you specify an id here +;google_tag_manager_id = + +# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set +;rudderstack_write_key = + +# Rudderstack data plane url, enabled only if rudderstack_write_key is also set +;rudderstack_data_plane_url = + +# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set +;rudderstack_sdk_url = + +# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config +;rudderstack_config_url = + +# Controls if the UI contains any links to user feedback forms +;feedback_links_enabled = true + +#################################### Security #################################### +[security] +# disable creation of admin user on first start of grafana +;disable_initial_admin_creation = false + +# default admin user, created on startup +;admin_user = admin + +# default admin password, can be changed before first start of grafana, or in profile settings +;admin_password = admin + +# used for signing +;secret_key = SW2YcwTIb9zpOOhoPsMm + +# current key provider used for envelope encryption, default to static value specified by secret_key +;encryption_provider = secretKey.v1 + +# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 +;available_encryption_providers = + +# disable gravatar profile images +;disable_gravatar = false + +# data source proxy whitelist (ip_or_domain:port separated by spaces) +;data_source_proxy_whitelist = + +# disable protection against brute force login attempts +;disable_brute_force_login_protection = false + +# set to true if you host.behind HTTPS. default is false. +;cookie_secure = false + +# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" +;cookie_samesite = lax + +# set to true if you want to allow browsers to render.in a ,