From a9aa10cb3e8737e98e2209a27e2938f1c21d71b7 Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Wed, 6 Jul 2022 14:35:29 +0200 Subject: [PATCH] feat(test): adding tests for the "File" struct feat(test): moving util funcs into utils module --- data/config/go-test-conf.json | 66 +++++++++++++++++++ data/templates/go-test-go.tpl | 13 ++++ data/templates/go-test-hcl.pktpl.hcl | 12 ++++ data/templates/loki-local-config.tpl | 8 +-- pkg/templater/files.go | 94 ++-------------------------- pkg/templater/files_test.go | 28 +++++++++ pkg/utils/templates.go | 89 ++++++++++++++++++++++++++ pkg/utils/templates_test.go | 34 ++++++++++ 8 files changed, 252 insertions(+), 92 deletions(-) create mode 100644 data/config/go-test-conf.json create mode 100644 data/templates/go-test-go.tpl create mode 100644 data/templates/go-test-hcl.pktpl.hcl create mode 100644 pkg/templater/files_test.go create mode 100644 pkg/utils/templates.go create mode 100644 pkg/utils/templates_test.go diff --git a/data/config/go-test-conf.json b/data/config/go-test-conf.json new file mode 100644 index 0000000..bddc09d --- /dev/null +++ b/data/config/go-test-conf.json @@ -0,0 +1,66 @@ +{ + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "group": "grafana", + "mode": "600", + "owner": "loki", + "service": "loki", + "source": "loki-local-config.pktpl.hcl" + } + ], + "Daemons": { + "Loki": { + "enabled": true, + "name": "loki" + } + }, + "Packages": { + "loki": { + "action": "install", + "name": "loki" + }, + "nodeExporter": { + "action": "install", + "name": "prometheus-node-exporter" + }, + "promtail": { + "action": "install", + "name": "loki-promtail" + } + }, + "Repositories": { + "AlpineTesting": { + "enabled": true, + "name": "testing", + "type": "apk", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing" + } + }, + "Users": { + "loki": { + "group": "grafana", + "home": "/srv/loki", + "shell": "/bin/nologin", + "username": "loki" + } + }, + "Vars": { + "AlertManagerURL": "http://localhost:9092", + "AuthEnabled": false, + "GRPCPort": "9095", + "Group": "grafana", + "HTTPPort": "3099", + "LogLevel": "error", + "ObjectStore": "filesystem", + "S2": { + "APIKey": "", + "APISecretKey": "", + "BucketName": "", + "URL": "" + }, + "SharedStore": "filesystem", + "StorageRoot": "/var/loki", + "User": "loki" + } + } \ No newline at end of file diff --git a/data/templates/go-test-go.tpl b/data/templates/go-test-go.tpl new file mode 100644 index 0000000..953cb01 --- /dev/null +++ b/data/templates/go-test-go.tpl @@ -0,0 +1,13 @@ +### Go template test ### +{{ if .Vars.AuthEnabled }} +auth_enabled: true +{{ else }} +auth_enabled: false +{{ end }} + +server: + http_listen_port: {{ .Vars.HTTPPort }} + grpc_listen_port: {{ .Vars.GRPCPort }} + log_level: {{ .Vars.LogLevel }} + +### End Go template test ### \ No newline at end of file diff --git a/data/templates/go-test-hcl.pktpl.hcl b/data/templates/go-test-hcl.pktpl.hcl new file mode 100644 index 0000000..1671af5 --- /dev/null +++ b/data/templates/go-test-hcl.pktpl.hcl @@ -0,0 +1,12 @@ +### HCL2 Template test ### +%{ if Vars.AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } + +server: + http_listen_port: ${Vars.HTTPPort} + grpc_listen_port: ${Vars.GRPCPort} + log_level: ${Vars.LogLevel} +### END HCL Template test ### diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl index 07cd8c8..df468ff 100644 --- a/data/templates/loki-local-config.tpl +++ b/data/templates/loki-local-config.tpl @@ -1,13 +1,13 @@ -{{ if .AuthEnabled }} +{{ if .Vars.AuthEnabled }} auth_enabled: true {{ else }} auth_enabled: false {{ end }} server: - http_listen_port: {{ .HTTPPort }} - grpc_listen_port: {{ .GRPCPort }} - log_level: {{ .LogLevel }} + http_listen_port: {{ .Vars.HTTPPort }} + grpc_listen_port: {{ .Vars.GRPCPort }} + log_level: {{ .Vars.LogLevel }} ingester: wal: diff --git a/pkg/templater/files.go b/pkg/templater/files.go index c5428ab..f59853c 100644 --- a/pkg/templater/files.go +++ b/pkg/templater/files.go @@ -5,16 +5,8 @@ import ( "path/filepath" "strconv" - "bytes" - encjson "encoding/json" "fmt" "os" - "text/template" - - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - ctyjson "github.com/zclconf/go-cty/cty/json" "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" ) @@ -27,19 +19,20 @@ type ConfigFile struct { Owner string `json:"owner"` // The configuration file owner Service string `json:"service"` // Service to restart after configuration generation Group string `json:"group"` // The configuration file group owner + TemplateDir string } // Generate the configuration file from the template (hcl or json) func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error { var template string + cf.TemplateDir = templateDir dest := filepath.Join(root, cf.Destination) - source := filepath.Join(templateDir, cf.Source) intMod, err := strconv.ParseInt(cf.Mode, 8, 64) if err != nil { return (err) } - template, err = cf.ProcessTemplate(source, values) + template, err = cf.ProcessTemplate(root, values) if err != nil { return fmt.Errorf("Process templates failed with error: %v", err) } @@ -57,97 +50,22 @@ func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) e } // Process the template with the provided values -func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) { +func (cf *ConfigFile) ProcessTemplate(root string, values []byte) (string, error) { var result string var err error if cf.TemplateType == "hcl" { // The template is an hcl template so we call processHCLTemplate - result, err = cf.processHCLTemplate(source, values) + result, err = utils.ProcessHCLTemplate(filepath.Join(cf.TemplateDir, cf.Source), values) if err != nil { return "", fmt.Errorf("Process HCL template failed with error: %v", err) } } else if cf.TemplateType == "go" { // The template is a go template so we call processGoTemplate - result, err = cf.processGoTemplate(source, values) + result, err = utils.ProcessGoTemplate(filepath.Join(cf.TemplateDir, cf.Source), values) if err != nil { return "", fmt.Errorf("Process GO template failed with error: %v", err) } } return result, nil } - -// The actual template processing for Go templates -func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) { - - // The JSON configuration - var confData map[string]interface{} - var res bytes.Buffer - - err := encjson.Unmarshal(configValues, &confData) - utils.CheckErr(err) - - // Read the template - data, err := os.ReadFile(file) - utils.CheckErr(err) - - tpl, err := template.New("conf").Parse(string(data)) - utils.CheckErr(err) - - utils.CheckErr(tpl.Execute(&res, confData["Config"])) - - return res.String(), nil -} - -// The actual template processing for HCL templates -func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) { - - fct, err := os.ReadFile(file) - utils.CheckErr(err) - - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) - utils.CheckDiags(diags) - - // Retrieve values from JSON - var varsVal cty.Value - ctyType, err := ctyjson.ImpliedType(config) - if err != nil { - return "", err - /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) - if diags.HasErrors() { - panic(diags.Error()) - } - varsVal, diags = cexpr.Value(&hcl.EvalContext{}) - fmt.Println(cexpr.Variables()) - checkDiags(diags) - */ - } else { - varsVal, err = ctyjson.Unmarshal(config, ctyType) - utils.CheckErr(err) - } - - ctx := &hcl.EvalContext{ - Variables: varsVal.AsValueMap(), - } - - for n := range ctx.Variables { - if !hclsyntax.ValidIdentifier(n) { - return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n) - } - } - - for _, traversal := range expr.Variables() { - root := traversal.RootName() - if _, ok := ctx.Variables[root]; !ok { - return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()) - } - } - - val, diags := expr.Value(ctx) - if diags.HasErrors() { - return "", diags - } - - return val.AsString(), nil -} diff --git a/pkg/templater/files_test.go b/pkg/templater/files_test.go new file mode 100644 index 0000000..bcc89ab --- /dev/null +++ b/pkg/templater/files_test.go @@ -0,0 +1,28 @@ +package templater + +import ( + "io/ioutil" + "testing" +) + +func TestProcessTemplate(t *testing.T) { + + goFile := ConfigFile{ + Destination: "/loki-config.test", + Source: "loki-local-config.tpl", + TemplateType: "go", + Mode: "700", + TemplateDir: "../../data/templates/", + } + + values, err := ioutil.ReadFile("../../data/config/loki-stack.json") + if err != nil { + t.Error(err) + } + + data, err := goFile.ProcessTemplate("/tmp/", values) + if err != nil { + t.Errorf(err.Error()) + } + t.Log(data) +} diff --git a/pkg/utils/templates.go b/pkg/utils/templates.go new file mode 100644 index 0000000..d5204df --- /dev/null +++ b/pkg/utils/templates.go @@ -0,0 +1,89 @@ +package utils + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "text/template" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" +) + +// The actual template processing for Go templates +func ProcessGoTemplate(file string, configValues []byte) (string, error) { + + // The JSON configuration + var confData map[string]interface{} + var res bytes.Buffer + + err := json.Unmarshal(configValues, &confData) + CheckErr(err) + + // Read the template + templateData, err := os.ReadFile(file) + CheckErr(err) + + tpl, err := template.New("conf").Parse(string(templateData)) + CheckErr(err) + + CheckErr(tpl.Execute(&res, confData)) + + return res.String(), nil +} + +// The actual template processing for HCL templates +func ProcessHCLTemplate(file string, config []byte) (string, error) { + + fct, err := os.ReadFile(file) + CheckErr(err) + + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + CheckDiags(diags) + + // Retrieve values from JSON + var varsVal cty.Value + ctyType, err := ctyjson.ImpliedType(config) + if err != nil { + return "", err + /* Maybe one day + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) + if diags.HasErrors() { + panic(diags.Error()) + } + varsVal, diags = cexpr.Value(&hcl.EvalContext{}) + fmt.Println(cexpr.Variables()) + checkDiags(diags) + */ + } else { + varsVal, err = ctyjson.Unmarshal(config, ctyType) + CheckErr(err) + } + + ctx := &hcl.EvalContext{ + Variables: varsVal.AsValueMap(), + } + + for n := range ctx.Variables { + if !hclsyntax.ValidIdentifier(n) { + return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n) + } + } + + for _, traversal := range expr.Variables() { + root := traversal.RootName() + if _, ok := ctx.Variables[root]; !ok { + return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()) + } + } + + val, diags := expr.Value(ctx) + if diags.HasErrors() { + return "", diags + } + + return val.AsString(), nil +} diff --git a/pkg/utils/templates_test.go b/pkg/utils/templates_test.go new file mode 100644 index 0000000..2203a5a --- /dev/null +++ b/pkg/utils/templates_test.go @@ -0,0 +1,34 @@ +package utils + +import ( + "io/ioutil" + "testing" +) + +func TestProcessHCLTemplate(t *testing.T) { + // load the Full configuration from a file + values, err := ioutil.ReadFile("../../data/config/go-test-conf.json") + if err != nil { + t.Error(err) + } + + data, err := ProcessHCLTemplate("../../data/templates/go-test-hcl.pktpl.hcl", values) + if err != nil { + t.Errorf(err.Error()) + } + t.Logf("%s", data) +} + +func TestProcessGoTemplate(t *testing.T) { + // load values from testing json file + values, err := ioutil.ReadFile("../../data/config/go-test-conf.json") + if err != nil { + t.Error(err) + } + + data, err := ProcessGoTemplate("../../data/templates/go-test-go.tpl", values) + if err != nil { + t.Error(err) + } + t.Logf("%s", data) +}