Merge pull request 'feat(api): adding api server code' (#1) from feat/api into dev

Reviewed-on: #1

Makefile

@@ -2,8 +2,9 @@ LINT_ARGS ?= ./...
 DESTDIR ?= "/usr/local"
 
 bin:
-	GOOS=linux go build -o bin/templater-linux main.go
+	GOOS=linux go build -o bin/templater-linux cmd/templater.go
 	upx bin/templater-linux
+	upx bin/templaster-server
 
 install:
 	cp bin/templater-linux $(DESTDIR)/bin/templater

api/main.go

@@ -0,0 +1,44 @@
+package api
+
+import (
+	"net/http"
+
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
+	"github.com/gin-gonic/gin"
+)
+
+type Template struct {
+	Type    string
+	Content string
+	Config  string
+}
+
+func Generate(c *gin.Context) {
+	var template Template
+
+	err := c.Request.ParseForm()
+	if err != nil {
+		c.String(500, err.Error())
+	}
+
+	err = c.ShouldBindJSON(&template)
+	if err != nil {
+		c.String(500, err.Error())
+		return
+	}
+
+	templateType := template.Type
+	templateFile := template.Content
+	config := []byte(template.Config)
+	res := ""
+	if templateType == "go" {
+		res = templater.ProcessGoTemplate(templateFile, config)
+		c.JSON(http.StatusOK, gin.H{"data": res})
+	} else if templateType == "hcl" {
+		res = templater.ProcessHCLTemplate(templateFile, config)
+		c.JSON(http.StatusOK, gin.H{"data": res})
+	} else {
+		c.JSON(http.StatusBadRequest, gin.H{"data": "Unkown template type"})
+	}
+
+}

cmd/templater.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"forge.cadoles.com/pcaseiro/templatefile/api"
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
+	"github.com/alexflint/go-arg"
+	"github.com/gin-gonic/gin"
+)
+
+func Daemon(port int) (err error) {
+	r := gin.Default()
+
+	r.POST("/generate", api.Generate)
+
+	err = r.Run(fmt.Sprintf("0.0.0.0:%d", port)) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
+	if err != nil {
+		return (err)
+	}
+	return nil
+}
+
+func main() {
+
+	var args struct {
+		Daemon bool   `arg:"-d,--daemon,env:TEMPLATER_DAEMON" default:"false" help:"Enable api server"`
+		Port   int    `arg:"-p,--port,env:TEMPLATER_PORT" default:"8080" help:"Listening port for the api server"`
+		Type   string `arg:"-t,--type,env:TEMPLATE_TYPE" default:"hcl" help:"Template type (go/template or hcl)"`
+		Output string `arg:"-o,--output,env:TEMPLATER_OUTPUT" default:"stdout" help:"Destination of the result (stdout or file path)"`
+		Config string `arg:"-c,--config,env:TEMPLATE_CONFIG" help:"Configuration values"`
+		File   string `arg:"-f,--template-file,env:TEMPLATE_FILE" help:"Template file path"`
+	}
+
+	arg.MustParse(&args)
+
+	if args.Daemon {
+		err := Daemon(args.Port)
+		if err != nil {
+			panic(err)
+		}
+	} else {
+		var config []byte
+		templateType := args.Type
+		templateFile := args.File
+		output := args.Output
+
+		if _, err := os.Stat(args.Config); err == nil {
+			config, err = os.ReadFile(args.Config)
+			if err != nil {
+				panic(err)
+			}
+		} else {
+			config = []byte(args.Config)
+		}
+
+		result := ""
+		if templateType == "go" {
+			result = templater.ProcessGoTemplate(templateFile, config)
+		} else if templateType == "hcl" {
+			result = templater.ProcessHCLTemplate(templateFile, config)
+		} else {
+			panic(fmt.Errorf("Unsupported template type"))
+		}
+		if output == "stdout" {
+			fmt.Printf("%s", result)
+		} else {
+			err := os.WriteFile(output, []byte(result), 0644)
+			if err != nil {
+				panic(err)
+			}
+		}
+	}
+
+}

data/config/test.json

@@ -0,0 +1,26 @@
+{
+  "Name": "loki",
+  "ConfigFiles": [
+    {
+      "destination": "/etc/loki/loki-local-config.yaml",
+      "source": "loki-local-config.pktpl.hcl",
+      "mod": "600"
+    }
+  ],
+  "AuthEnabled": false,
+  "User": "loki",
+  "Group": "grafana",
+  "HTTPPort": "3100",
+  "GRPCPort": "9096",
+  "AlertManagerURL": "http://localhost:9093",
+  "StorageRoot": "/var/loki",
+  "SharedStore": "filesystem",
+  "ObjectStore": "filesystem",
+  "LogLevel": "error",
+  "S3": {
+      "URL": "",
+      "BucketName": "",
+      "APIKey": "",
+      "APISecretKey": ""
+  }
+}

data/templates/loki-local-config.pktpl.hcl

@@ -0,0 +1,82 @@
+%{ if AuthEnabled ~}
+auth_enabled: true
+%{ else }
+auth_enabled: false
+%{ endif }
+
+server:
+  http_listen_port: ${HTTPPort}
+  grpc_listen_port: ${GRPCPort}
+  log_level: ${LogLevel}
+
+ingester:
+  wal:
+    enabled: true
+    dir: ${StorageRoot}/wal
+    flush_on_shutdown: true
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb-shipper
+    object_store: ${ObjectStore}
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+ boltdb_shipper:
+   active_index_directory: ${StorageRoot}/index
+   shared_store: ${SharedStore}
+   cache_location: ${StorageRoot}/cache
+   cache_ttl: 168h
+
+%{ if ObjectStore == "filesystem" ~}
+  filesystem:
+    directory: ${StorageRoot}/chunks
+%{ else }
+  aws:
+    s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName}
+    s3forcepathstyle: true
+%{ endif }
+
+compactor:
+  shared_store: ${SharedStore}
+  working_directory: ${StorageRoot}/compactor
+  compaction_interval: 10m
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: ${StorageRoot}/rules
+  rule_path: ${StorageRoot}/rules
+  alertmanager_url: ${AlertManagerURL}
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true

data/templates/loki-local-config.tpl

@@ -0,0 +1,82 @@
+{{ if .AuthEnabled  }}
+auth_enabled: true
+{{ else  }}
+auth_enabled: false
+{{ end  }}
+
+server:
+  http_listen_port: {{ .HTTPPort }}
+  grpc_listen_port: {{ .GRPCPort }}
+  log_level: {{ .LogLevel }}
+
+ingester:
+  wal:
+    enabled: true
+    dir: {{ .StorageRoot }}/wal
+    flush_on_shutdown: true
+  lifecycler:
+    address: 127.0.0.1
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+    final_sleep: 0s
+  chunk_idle_period: 1h       # Any chunk not receiving new logs in this time will be flushed
+  max_chunk_age: 1h           # All chunks will be flushed when they hit this age, default is 1h
+  chunk_target_size: 1048576  # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
+  chunk_retain_period: 30s    # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
+  max_transfer_retries: 0     # Chunk transfers disabled
+
+schema_config:
+  configs:
+  - from: 2020-05-15
+    store: boltdb-shipper
+    object_store: {{ .ObjectStore }}
+    schema: v11
+    index:
+      prefix: index_
+      period: 24h
+
+storage_config:
+ boltdb_shipper:
+   active_index_directory: {{ .StorageRoot }}/index
+   shared_store: {{ .SharedStore }}
+   cache_location: {{ .StorageRoot }}/cache
+   cache_ttl: 168h
+
+{{ if eq (.ObjectStore) ("filesystem") }}
+  filesystem:
+    directory: {{ .StorageRoot }}/chunks
+{{ else  }}
+  aws:
+    s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}}
+    s3forcepathstyle: true
+{{ end  }}
+
+compactor:
+  shared_store: {{ .SharedStore }}
+  working_directory: {{ .StorageRoot }}/compactor
+  compaction_interval: 10m
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+
+chunk_store_config:
+  max_look_back_period: 0s
+
+table_manager:
+  retention_deletes_enabled: false
+  retention_period: 0s
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: {{ .StorageRoot }}/rules
+  rule_path: {{ .StorageRoot }}/rules
+  alertmanager_url: {{ .AlertManagerURL }}
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true

pkg/templater/main.go

@@ -1,6 +1,7 @@
-package main
+package templater
 
 import (
+	"bytes"
 	encjson "encoding/json"
 	"fmt"
 	"os"
@@ -10,45 +11,38 @@ import (
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	"github.com/zclconf/go-cty/cty"
 	ctyjson "github.com/zclconf/go-cty/cty/json"
+
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
 )
 
-func checkErr(e error) {
+func ProcessGoTemplate(file string, config []byte) string {
-	if e != nil {
-		panic(e)
-	}
-}
-
-func checkDiags(diag hcl.Diagnostics) {
-	if diag.HasErrors() {
-		panic(diag.Error())
-	}
-}
-
-func processGoTemplate(file string, config []byte) {
 
 	// The JSON configuration
 	var confData map[string]interface{}
+	var res bytes.Buffer
+
 	err := encjson.Unmarshal(config, &confData)
-	checkErr(err)
+	utils.CheckErr(err)
 
 	// Read the template
 	data, err := os.ReadFile(file)
-	checkErr(err)
+	utils.CheckErr(err)
 
 	tpl, err := template.New("conf").Parse(string(data))
-	checkErr(err)
+	utils.CheckErr(err)
 
-	checkErr(tpl.Execute(os.Stdout, confData))
+	utils.CheckErr(tpl.Execute(&res, confData))
 
+	return res.String()
 }
 
-func processHCLTemplate(file string, config []byte) {
+func ProcessHCLTemplate(file string, config []byte) string {
 
 	fct, err := os.ReadFile(file)
-	checkErr(err)
+	utils.CheckErr(err)
 
-	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1})
+	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
-	checkDiags(diags)
+	utils.CheckDiags(diags)
 
 	// Retrieve values from JSON
 	var varsVal cty.Value
@@ -56,7 +50,7 @@ func processHCLTemplate(file string, config []byte) {
 	if err != nil {
 		panic(err)
 		/* Maybe one day
-		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1})
+		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
 		if diags.HasErrors() {
 			panic(diags.Error())
 		}
@@ -66,7 +60,7 @@ func processHCLTemplate(file string, config []byte) {
 		*/
 	} else {
 		varsVal, err = ctyjson.Unmarshal(config, ctyType)
-		checkErr(err)
+		utils.CheckErr(err)
 	}
 
 	ctx := &hcl.EvalContext{
@@ -91,20 +85,5 @@ func processHCLTemplate(file string, config []byte) {
 		panic(diags.Error())
 	}
 
-	fmt.Printf("%s", val.AsString())
+	return val.AsString()
-}
-
-func main() {
-	// The template to process
-	templateType := os.Args[1]
-	templateFile := os.Args[2]
-	config := []byte(os.Args[3])
-
-	if templateType == "go" {
-		processGoTemplate(templateFile, config)
-	} else if templateType == "hcl" {
-		processHCLTemplate(templateFile, config)
-	} else {
-		panic(fmt.Errorf("Unsupported template type"))
-	}
 }

pkg/utils/main.go

@@ -0,0 +1,17 @@
+package utils
+
+import (
+	"github.com/hashicorp/hcl/v2"
+)
+
+func CheckErr(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+func CheckDiags(diag hcl.Diagnostics) {
+	if diag.HasErrors() {
+		panic(diag.Error())
+	}
+}