From 0219bbc5fb62fa7c12611aec5f2b85611304dc28 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Thu, 24 Mar 2022 00:49:23 +0100 Subject: [PATCH] feat(api): adding api server code templater can be used as a commmand line tool or as an api server. --- Makefile | 3 +- api/main.go | 44 ++++++++++++ cmd/templater.go | 76 ++++++++++++++++++++ data/config/test.json | 26 +++++++ data/templates/loki-local-config.pktpl.hcl | 82 ++++++++++++++++++++++ data/templates/loki-local-config.tpl | 82 ++++++++++++++++++++++ main.go => pkg/templater/main.go | 59 +++++----------- pkg/utils/main.go | 17 +++++ 8 files changed, 348 insertions(+), 41 deletions(-) create mode 100644 api/main.go create mode 100644 cmd/templater.go create mode 100644 data/config/test.json create mode 100644 data/templates/loki-local-config.pktpl.hcl create mode 100644 data/templates/loki-local-config.tpl rename main.go => pkg/templater/main.go (66%) create mode 100644 pkg/utils/main.go diff --git a/Makefile b/Makefile index b2d52ac..254cad2 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,9 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux main.go + GOOS=linux go build -o bin/templater-linux cmd/templater.go upx bin/templater-linux + upx bin/templaster-server install: cp bin/templater-linux $(DESTDIR)/bin/templater diff --git a/api/main.go b/api/main.go new file mode 100644 index 0000000..7c12c62 --- /dev/null +++ b/api/main.go @@ -0,0 +1,44 @@ +package api + +import ( + "net/http" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/gin-gonic/gin" +) + +type Template struct { + Type string + Content string + Config string +} + +func Generate(c *gin.Context) { + var template Template + + err := c.Request.ParseForm() + if err != nil { + c.String(500, err.Error()) + } + + err = c.ShouldBindJSON(&template) + if err != nil { + c.String(500, err.Error()) + return + } + + templateType := template.Type + templateFile := template.Content + config := []byte(template.Config) + res := "" + if templateType == "go" { + res = templater.ProcessGoTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else if templateType == "hcl" { + res = templater.ProcessHCLTemplate(templateFile, config) + c.JSON(http.StatusOK, gin.H{"data": res}) + } else { + c.JSON(http.StatusBadRequest, gin.H{"data": "Unkown template type"}) + } + +} diff --git a/cmd/templater.go b/cmd/templater.go new file mode 100644 index 0000000..2151958 --- /dev/null +++ b/cmd/templater.go @@ -0,0 +1,76 @@ +package main + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/api" + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" + "github.com/gin-gonic/gin" +) + +func Daemon(port int) (err error) { + r := gin.Default() + + r.POST("/generate", api.Generate) + + err = r.Run(fmt.Sprintf("0.0.0.0:%d", port)) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") + if err != nil { + return (err) + } + return nil +} + +func main() { + + var args struct { + Daemon bool `arg:"-d,--daemon,env:TEMPLATER_DAEMON" default:"false"` + Port int `arg:"-p,--port,env:TEMPLATER_PORT" default:"8080"` + Type string `arg:"-t,--type,env:TEMPLATE_TYPE" default:"hcl"` + File string `arg:"-f,--template-file,env:TEMPLATE_FILE"` + Output string `arg:"-o,--output,env:TEMPLATER_OUTPUT" default:"stdout"` + Config string `arg:"-c,--config,env:TEMPLATE_CONFIG"` + } + + arg.MustParse(&args) + + if args.Daemon { + err := Daemon(args.Port) + if err != nil { + panic(err) + } + } else { + var config []byte + templateType := args.Type + templateFile := args.File + output := args.Output + + if _, err := os.Stat(args.Config); err == nil { + config, err = os.ReadFile(args.Config) + if err != nil { + panic(err) + } + } else { + config = []byte(args.Config) + } + + result := "" + if templateType == "go" { + result = templater.ProcessGoTemplate(templateFile, config) + } else if templateType == "hcl" { + result = templater.ProcessHCLTemplate(templateFile, config) + } else { + panic(fmt.Errorf("Unsupported template type")) + } + if output == "stdout" { + fmt.Printf("%s", result) + } else { + err := os.WriteFile(output, []byte(result), 0644) + if err != nil { + panic(err) + } + } + } + +} diff --git a/data/config/test.json b/data/config/test.json new file mode 100644 index 0000000..14035cb --- /dev/null +++ b/data/config/test.json @@ -0,0 +1,26 @@ +{ + "Name": "loki", + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mod": "600" + } + ], + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } +} \ No newline at end of file diff --git a/data/templates/loki-local-config.pktpl.hcl b/data/templates/loki-local-config.pktpl.hcl new file mode 100644 index 0000000..9fd10f7 --- /dev/null +++ b/data/templates/loki-local-config.pktpl.hcl @@ -0,0 +1,82 @@ +%{ if AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } + +server: + http_listen_port: ${HTTPPort} + grpc_listen_port: ${GRPCPort} + log_level: ${LogLevel} + +ingester: + wal: + enabled: true + dir: ${StorageRoot}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: ${ObjectStore} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: ${StorageRoot}/index + shared_store: ${SharedStore} + cache_location: ${StorageRoot}/cache + cache_ttl: 168h + +%{ if ObjectStore == "filesystem" ~} + filesystem: + directory: ${StorageRoot}/chunks +%{ else } + aws: + s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName} + s3forcepathstyle: true +%{ endif } + +compactor: + shared_store: ${SharedStore} + working_directory: ${StorageRoot}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: ${StorageRoot}/rules + rule_path: ${StorageRoot}/rules + alertmanager_url: ${AlertManagerURL} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl new file mode 100644 index 0000000..07cd8c8 --- /dev/null +++ b/data/templates/loki-local-config.tpl @@ -0,0 +1,82 @@ +{{ if .AuthEnabled }} +auth_enabled: true +{{ else }} +auth_enabled: false +{{ end }} + +server: + http_listen_port: {{ .HTTPPort }} + grpc_listen_port: {{ .GRPCPort }} + log_level: {{ .LogLevel }} + +ingester: + wal: + enabled: true + dir: {{ .StorageRoot }}/wal + flush_on_shutdown: true + lifecycler: + address: 127.0.0.1 + ring: + kvstore: + store: inmemory + replication_factor: 1 + final_sleep: 0s + chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed + max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h + chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) + max_transfer_retries: 0 # Chunk transfers disabled + +schema_config: + configs: + - from: 2020-05-15 + store: boltdb-shipper + object_store: {{ .ObjectStore }} + schema: v11 + index: + prefix: index_ + period: 24h + +storage_config: + boltdb_shipper: + active_index_directory: {{ .StorageRoot }}/index + shared_store: {{ .SharedStore }} + cache_location: {{ .StorageRoot }}/cache + cache_ttl: 168h + +{{ if eq (.ObjectStore) ("filesystem") }} + filesystem: + directory: {{ .StorageRoot }}/chunks +{{ else }} + aws: + s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} + s3forcepathstyle: true +{{ end }} + +compactor: + shared_store: {{ .SharedStore }} + working_directory: {{ .StorageRoot }}/compactor + compaction_interval: 10m + +limits_config: + reject_old_samples: true + reject_old_samples_max_age: 168h + +chunk_store_config: + max_look_back_period: 0s + +table_manager: + retention_deletes_enabled: false + retention_period: 0s + +ruler: + storage: + type: local + local: + directory: {{ .StorageRoot }}/rules + rule_path: {{ .StorageRoot }}/rules + alertmanager_url: {{ .AlertManagerURL }} + ring: + kvstore: + store: inmemory + enable_api: true \ No newline at end of file diff --git a/main.go b/pkg/templater/main.go similarity index 66% rename from main.go rename to pkg/templater/main.go index 046fe41..f1fba18 100644 --- a/main.go +++ b/pkg/templater/main.go @@ -1,6 +1,7 @@ -package main +package templater import ( + "bytes" encjson "encoding/json" "fmt" "os" @@ -10,45 +11,38 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" ) -func checkErr(e error) { - if e != nil { - panic(e) - } -} - -func checkDiags(diag hcl.Diagnostics) { - if diag.HasErrors() { - panic(diag.Error()) - } -} - -func processGoTemplate(file string, config []byte) { +func ProcessGoTemplate(file string, config []byte) string { // The JSON configuration var confData map[string]interface{} + var res bytes.Buffer + err := encjson.Unmarshal(config, &confData) - checkErr(err) + utils.CheckErr(err) // Read the template data, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) tpl, err := template.New("conf").Parse(string(data)) - checkErr(err) + utils.CheckErr(err) - checkErr(tpl.Execute(os.Stdout, confData)) + utils.CheckErr(tpl.Execute(&res, confData)) + return res.String() } -func processHCLTemplate(file string, config []byte) { +func ProcessHCLTemplate(file string, config []byte) string { fct, err := os.ReadFile(file) - checkErr(err) + utils.CheckErr(err) - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1}) - checkDiags(diags) + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) // Retrieve values from JSON var varsVal cty.Value @@ -56,7 +50,7 @@ func processHCLTemplate(file string, config []byte) { if err != nil { panic(err) /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1}) + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) if diags.HasErrors() { panic(diags.Error()) } @@ -66,7 +60,7 @@ func processHCLTemplate(file string, config []byte) { */ } else { varsVal, err = ctyjson.Unmarshal(config, ctyType) - checkErr(err) + utils.CheckErr(err) } ctx := &hcl.EvalContext{ @@ -91,20 +85,5 @@ func processHCLTemplate(file string, config []byte) { panic(diags.Error()) } - fmt.Printf("%s", val.AsString()) -} - -func main() { - // The template to process - templateType := os.Args[1] - templateFile := os.Args[2] - config := []byte(os.Args[3]) - - if templateType == "go" { - processGoTemplate(templateFile, config) - } else if templateType == "hcl" { - processHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) - } + return val.AsString() } diff --git a/pkg/utils/main.go b/pkg/utils/main.go new file mode 100644 index 0000000..8ea276d --- /dev/null +++ b/pkg/utils/main.go @@ -0,0 +1,17 @@ +package utils + +import ( + "github.com/hashicorp/hcl/v2" +) + +func CheckErr(e error) { + if e != nil { + panic(e) + } +} + +func CheckDiags(diag hcl.Diagnostics) { + if diag.HasErrors() { + panic(diag.Error()) + } +}