pcaseiro
/
templatefile
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 4 Wiki Activity

Merge bootstraper #3

Merged
pcaseiro merged 11 commits from dev into master 2023-01-12 11:46:21 +01:00
Conversation 0 Commits 11 Files Changed 37 +348 -41
8 changed files with 348 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 5085bd4d69 - Show all commits

3
Makefile
View File

@ -2,8 +2,9 @@ LINT_ARGS ?= ./...
DESTDIR ?= "/usr/local"
bin:
GOOS=linux go build -o bin/templater-linux main.go
GOOS=linux go build -o bin/templater-linux cmd/templater.go
upx bin/templater-linux
upx bin/templaster-server
install:
cp bin/templater-linux $(DESTDIR)/bin/templater

44
api/main.go Normal file
View File

@ -0,0 +1,44 @@
package api
import (
"net/http"
"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
"github.com/gin-gonic/gin"
)
type Template struct {
Type string
Content string
Config string
}
func Generate(c *gin.Context) {
var template Template
err := c.Request.ParseForm()
if err != nil {
c.String(500, err.Error())
}
err = c.ShouldBindJSON(&template)
if err != nil {
c.String(500, err.Error())
return
}
templateType := template.Type
templateFile := template.Content
config := []byte(template.Config)
res := ""
if templateType == "go" {
res = templater.ProcessGoTemplate(templateFile, config)
c.JSON(http.StatusOK, gin.H{"data": res})
} else if templateType == "hcl" {
res = templater.ProcessHCLTemplate(templateFile, config)
c.JSON(http.StatusOK, gin.H{"data": res})
} else {
c.JSON(http.StatusBadRequest, gin.H{"data": "Unkown template type"})
}
}

76
cmd/templater.go Normal file
View File

@ -0,0 +1,76 @@
package main
import (
"fmt"
"os"
"forge.cadoles.com/pcaseiro/templatefile/api"
"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
"github.com/alexflint/go-arg"
"github.com/gin-gonic/gin"
)
func Daemon(port int) (err error) {
r := gin.Default()
r.POST("/generate", api.Generate)
err = r.Run(fmt.Sprintf("0.0.0.0:%d", port)) // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080")
if err != nil {
return (err)
}
return nil
}
func main() {
var args struct {
Daemon bool `arg:"-d,--daemon,env:TEMPLATER_DAEMON" default:"false" help:"Enable api server"`
Port int `arg:"-p,--port,env:TEMPLATER_PORT" default:"8080" help:"Listening port for the api server"`
Type string `arg:"-t,--type,env:TEMPLATE_TYPE" default:"hcl" help:"Template type (go/template or hcl)"`
Output string `arg:"-o,--output,env:TEMPLATER_OUTPUT" default:"stdout" help:"Destination of the result (stdout or file path)"`
Config string `arg:"-c,--config,env:TEMPLATE_CONFIG" help:"Configuration values"`
File string `arg:"-f,--template-file,env:TEMPLATE_FILE" help:"Template file path"`
}
arg.MustParse(&args)
if args.Daemon {
err := Daemon(args.Port)
if err != nil {
panic(err)
}
} else {
var config []byte
templateType := args.Type
templateFile := args.File
output := args.Output
if _, err := os.Stat(args.Config); err == nil {
config, err = os.ReadFile(args.Config)
if err != nil {
panic(err)
}
} else {
config = []byte(args.Config)
}
result := ""
if templateType == "go" {
result = templater.ProcessGoTemplate(templateFile, config)
} else if templateType == "hcl" {
result = templater.ProcessHCLTemplate(templateFile, config)
} else {
panic(fmt.Errorf("Unsupported template type"))
}
if output == "stdout" {
fmt.Printf("%s", result)
} else {
err := os.WriteFile(output, []byte(result), 0644)
if err != nil {
panic(err)
}
}
}
}

26
data/config/test.json Normal file
View File

@ -0,0 +1,26 @@
{
"Name": "loki",
"ConfigFiles": [
{
"destination": "/etc/loki/loki-local-config.yaml",
"source": "loki-local-config.pktpl.hcl",
"mod": "600"
}
],
"AuthEnabled": false,
"User": "loki",
"Group": "grafana",
"HTTPPort": "3100",
"GRPCPort": "9096",
"AlertManagerURL": "http://localhost:9093",
"StorageRoot": "/var/loki",
"SharedStore": "filesystem",
"ObjectStore": "filesystem",
"LogLevel": "error",
"S3": {
"URL": "",
"BucketName": "",
"APIKey": "",
"APISecretKey": ""
}
}

82
data/templates/loki-local-config.pktpl.hcl Normal file
View File

@ -0,0 +1,82 @@
%{ if AuthEnabled ~}
auth_enabled: true
%{ else }
auth_enabled: false
%{ endif }
server:
http_listen_port: ${HTTPPort}
grpc_listen_port: ${GRPCPort}
log_level: ${LogLevel}
ingester:
wal:
enabled: true
dir: ${StorageRoot}/wal
flush_on_shutdown: true
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-05-15
store: boltdb-shipper
object_store: ${ObjectStore}
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: ${StorageRoot}/index
shared_store: ${SharedStore}
cache_location: ${StorageRoot}/cache
cache_ttl: 168h
%{ if ObjectStore == "filesystem" ~}
filesystem:
directory: ${StorageRoot}/chunks
%{ else }
aws:
s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName}
s3forcepathstyle: true
%{ endif }
compactor:
shared_store: ${SharedStore}
working_directory: ${StorageRoot}/compactor
compaction_interval: 10m
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
ruler:
storage:
type: local
local:
directory: ${StorageRoot}/rules
rule_path: ${StorageRoot}/rules
alertmanager_url: ${AlertManagerURL}
ring:
kvstore:
store: inmemory
enable_api: true

82
data/templates/loki-local-config.tpl Normal file
View File

@ -0,0 +1,82 @@
{{ if .AuthEnabled }}
auth_enabled: true
{{ else }}
auth_enabled: false
{{ end }}
server:
http_listen_port: {{ .HTTPPort }}
grpc_listen_port: {{ .GRPCPort }}
log_level: {{ .LogLevel }}
ingester:
wal:
enabled: true
dir: {{ .StorageRoot }}/wal
flush_on_shutdown: true
lifecycler:
address: 127.0.0.1
ring:
kvstore:
store: inmemory
replication_factor: 1
final_sleep: 0s
chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed
max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h
chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first
chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m)
max_transfer_retries: 0 # Chunk transfers disabled
schema_config:
configs:
- from: 2020-05-15
store: boltdb-shipper
object_store: {{ .ObjectStore }}
schema: v11
index:
prefix: index_
period: 24h
storage_config:
boltdb_shipper:
active_index_directory: {{ .StorageRoot }}/index
shared_store: {{ .SharedStore }}
cache_location: {{ .StorageRoot }}/cache
cache_ttl: 168h
{{ if eq (.ObjectStore) ("filesystem") }}
filesystem:
directory: {{ .StorageRoot }}/chunks
{{ else }}
aws:
s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}}
s3forcepathstyle: true
{{ end }}
compactor:
shared_store: {{ .SharedStore }}
working_directory: {{ .StorageRoot }}/compactor
compaction_interval: 10m
limits_config:
reject_old_samples: true
reject_old_samples_max_age: 168h
chunk_store_config:
max_look_back_period: 0s
table_manager:
retention_deletes_enabled: false
retention_period: 0s
ruler:
storage:
type: local
local:
directory: {{ .StorageRoot }}/rules
rule_path: {{ .StorageRoot }}/rules
alertmanager_url: {{ .AlertManagerURL }}
ring:
kvstore:
store: inmemory
enable_api: true

59
main.go → pkg/templater/main.go
View File

@ -1,6 +1,7 @@
package main
package templater
import (
"bytes"
encjson "encoding/json"
"fmt"
"os"
@ -10,45 +11,38 @@ import (
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
ctyjson "github.com/zclconf/go-cty/cty/json"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
)
func checkErr(e error) {
if e != nil {
panic(e)
}
}
func checkDiags(diag hcl.Diagnostics) {
if diag.HasErrors() {
panic(diag.Error())
}
}
func processGoTemplate(file string, config []byte) {
func ProcessGoTemplate(file string, config []byte) string {
// The JSON configuration
var confData map[string]interface{}
var res bytes.Buffer
err := encjson.Unmarshal(config, &confData)
checkErr(err)
utils.CheckErr(err)
// Read the template
data, err := os.ReadFile(file)
checkErr(err)
utils.CheckErr(err)
tpl, err := template.New("conf").Parse(string(data))
checkErr(err)
utils.CheckErr(err)
checkErr(tpl.Execute(os.Stdout, confData))
utils.CheckErr(tpl.Execute(&res, confData))
return res.String()
}
func processHCLTemplate(file string, config []byte) {
func ProcessHCLTemplate(file string, config []byte) string {
fct, err := os.ReadFile(file)
checkErr(err)
utils.CheckErr(err)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 1, Column: 1})
checkDiags(diags)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
utils.CheckDiags(diags)
// Retrieve values from JSON
var varsVal cty.Value
@ -56,7 +50,7 @@ func processHCLTemplate(file string, config []byte) {
if err != nil {
panic(err)
/* Maybe one day
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 1, Column: 1})
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
if diags.HasErrors() {
panic(diags.Error())
}
@ -66,7 +60,7 @@ func processHCLTemplate(file string, config []byte) {
*/
} else {
varsVal, err = ctyjson.Unmarshal(config, ctyType)
checkErr(err)
utils.CheckErr(err)
}
ctx := &hcl.EvalContext{
@ -91,20 +85,5 @@ func processHCLTemplate(file string, config []byte) {
panic(diags.Error())
}
fmt.Printf("%s", val.AsString())
}
func main() {
// The template to process
templateType := os.Args[1]
templateFile := os.Args[2]
config := []byte(os.Args[3])
if templateType == "go" {
processGoTemplate(templateFile, config)
} else if templateType == "hcl" {
processHCLTemplate(templateFile, config)
} else {
panic(fmt.Errorf("Unsupported template type"))
}
return val.AsString()
}

17
pkg/utils/main.go Normal file
View File

@ -0,0 +1,17 @@
package utils
import (
"github.com/hashicorp/hcl/v2"
)
func CheckErr(e error) {
if e != nil {
panic(e)
}
}
func CheckDiags(diag hcl.Diagnostics) {
if diag.HasErrors() {
panic(diag.Error())
}
}