From 0f7174ed6ff38aeea42fdede8de5cb8480685cda Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Thu, 24 Mar 2022 00:49:23 +0100 Subject: [PATCH] feat(api): adding api server code templater can be used as a commmand line tool or as an api server. --- Makefile | 2 +- api/generate.go | 1 + cmd/templater.go | 23 ++++++ data/config/test.json | 26 +++++++ data/templates/loki-local-config.pktpl.hcl | 82 ++++++++++++++++++++++ data/templates/loki-local-config.tpl | 82 ++++++++++++++++++++++ main.go => pkg/templater/main.go | 59 +++++----------- pkg/utils/main.go | 17 +++++ 8 files changed, 251 insertions(+), 41 deletions(-) create mode 100644 api/generate.go create mode 100644 cmd/templater.go create mode 100644 data/config/test.json create mode 100644 data/templates/loki-local-config.pktpl.hcl create mode 100644 data/templates/loki-local-config.tpl rename main.go => pkg/templater/main.go (66%) create mode 100644 pkg/utils/main.go diff --git a/Makefile b/Makefile index b2d52ac..7eae70a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux main.go + GOOS=linux go build -o bin/templater-linux cmd/templater.go upx bin/templater-linux install: diff --git a/api/generate.go b/api/generate.go new file mode 100644 index 0000000..778f64e --- /dev/null +++ b/api/generate.go @@ -0,0 +1 @@ +package api diff --git a/cmd/templater.go b/cmd/templater.go new file mode 100644 index 0000000..821202f --- /dev/null +++ b/cmd/templater.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" +) + +func main() { + // The template to process + templateType := os.Args[1] + templateFile := os.Args[2] + config := []byte(os.Args[3]) + + if templateType == "go" { + fmt.Printf("%s", templater.ProcessGoTemplate(templateFile, config)) + } else if templateType == "hcl" { + fmt.Printf("%s", templater.ProcessHCLTemplate(templateFile, config)) + } else { + panic(fmt.Errorf("Unsupported template type")) + } +} diff --git a/data/config/test.json b/data/config/test.json new file mode 100644 index 0000000..14035cb --- /dev/null +++ b/data/config/test.json @@ -0,0 +1,26 @@ +{ + "Name": "loki", + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mod": "600" + } + ], + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } +} \ No newline at end of file diff --git a/data/templates/loki-local-config.pktpl.hcl b/data/templates/loki-local-config.pktpl.hcl new file mode 100644 index 0000000..9fd10f7 --- /dev/null +++ b/data/templates/loki-local-config.pktpl.hcl @@ -0,0 +1,82 @@ +%{ if AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } + +server: + http_listen_port: ${HTTPPort} + grpc_listen_port: ${GRPCPort} + log_level: ${LogLevel} + +ingester: + wal: + enabled: true + dir: ${StorageRoot}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: ${ObjectStore} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: ${StorageRoot}/index + shared_store: ${SharedStore} + cache_location: ${StorageRoot}/cache + cache_ttl: 168h + +%{ if ObjectStore == "filesystem" ~} + filesystem: + directory: ${StorageRoot}/chunks +%{ else } + aws: + s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName} + s3forcepathstyle: true +%{ endif } + +compactor: + shared_store: ${SharedStore} + working_directory: ${StorageRoot}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: ${StorageRoot}/rules + rule_path: ${StorageRoot}/rules + alertmanager_url: ${AlertManagerURL} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl new file mode 100644 index 0000000..07cd8c8 --- /dev/null +++ b/data/templates/loki-local-config.tpl @@ -0,0 +1,82 @@ +{{ if .AuthEnabled }} +auth_enabled: true +{{ else }} +auth_enabled: false +{{ end }} + +server: + http_listen_port: {{ .HTTPPort }} + grpc_listen_port: {{ .GRPCPort }} + log_level: {{ .LogLevel }} + +ingester: + wal: + enabled: true + dir: {{ .StorageRoot }}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: {{ .ObjectStore }} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: {{ .StorageRoot }}/index + shared_store: {{ .SharedStore }} + cache_location: {{ .StorageRoot }}/cache + cache_ttl: 168h + +{{ if eq (.ObjectStore) ("filesystem") }} + filesystem: + directory: {{ .StorageRoot }}/chunks +{{ else }} + aws: + s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} + s3forcepathstyle: true +{{ end }} + +compactor: + shared_store: {{ .SharedStore }} + working_directory: {{ .StorageRoot }}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: {{ .StorageRoot }}/rules + rule_path: {{ .StorageRoot }}/rules + alertmanager_url: {{ .AlertManagerURL }} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/main.go b/pkg/templater/main.go similarity index 66% rename from main.go rename to pkg/templater/main.go index 046fe41..f1fba18 100644 --- a/main.go +++ b/pkg/templater/main.go @@ -1,6 +1,7 @@ -package main +package templater import ( + "bytes" encjson "encoding/json" "fmt" "os" @@ -10,45 +11,38 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" ) -func checkErr(e error) { - if e != nil { - panic(e) - } -} - -func checkDiags(diag hcl.Diagnostics) { - if diag.HasErrors() { - panic(diag.Error()) - } -} - -func processGoTemplate(file string, config []byte) { +func ProcessGoTemplate(file string, config []byte) string { // The JSON configuration var confData map[string]interface{} + var res bytes.Buffer + err := encjson.Unmarshal(config, &confData) - checkErr(err) + utils.CheckErr(err) // Read the template data, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) tpl, err := template.New("conf").Parse(string(data)) - checkErr(err) + utils.CheckErr(err) - checkErr(tpl.Execute(os.Stdout, confData)) + utils.CheckErr(tpl.Execute(&res, confData)) + return res.String() } -func processHCLTemplate(file string, config []byte) { +func ProcessHCLTemplate(file string, config []byte) string { fct, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1}) - checkDiags(diags) + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) // Retrieve values from JSON var varsVal cty.Value @@ -56,7 +50,7 @@ func processHCLTemplate(file string, config []byte) { if err != nil { panic(err) /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1}) + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) if diags.HasErrors() { panic(diags.Error()) } @@ -66,7 +60,7 @@ func processHCLTemplate(file string, config []byte) { */ } else { varsVal, err = ctyjson.Unmarshal(config, ctyType) - checkErr(err) + utils.CheckErr(err) } ctx := &hcl.EvalContext{ @@ -91,20 +85,5 @@ func processHCLTemplate(file string, config []byte) { panic(diags.Error()) } - fmt.Printf("%s", val.AsString()) -} - -func main() { - // The template to process - templateType := os.Args[1] - templateFile := os.Args[2] - config := []byte(os.Args[3]) - - if templateType == "go" { - processGoTemplate(templateFile, config) - } else if templateType == "hcl" { - processHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) - } + return val.AsString() } diff --git a/pkg/utils/main.go b/pkg/utils/main.go new file mode 100644 index 0000000..8ea276d --- /dev/null +++ b/pkg/utils/main.go @@ -0,0 +1,17 @@ +package utils + +import ( + "github.com/hashicorp/hcl/v2" +) + +func CheckErr(e error) { + if e != nil { + panic(e) + } +} + +func CheckDiags(diag hcl.Diagnostics) { + if diag.HasErrors() { + panic(diag.Error()) + } +}