feat(test): adding tests for the "File" struct

feat(test): moving util funcs into utils module

cmd/bootstraper/main.go

@@ -11,6 +11,7 @@ func main() {
 		Config            string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"`
 		TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"`
 		RootDirectory     string `arg:"-r,--root-dir,env:ROOT_DIR" help:"Generate files with this root instead of /" default:"/"`
+		DryRun            bool   `arg:"-d,--dry-run,env:DRY_RUN" help:"Dry run do not really complete actions" default:"false"`
 	}
 
 	arg.MustParse(&args)
@@ -21,7 +22,7 @@ func main() {
 	if err != nil {
 		panic(err)
 	}
-	if err = hostConfig.ManageServices(); err != nil {
+	if err = hostConfig.ManageServices(args.DryRun); err != nil {
 		panic(err)
 	}
 }

data/config/go-test-conf.json

@@ -0,0 +1,66 @@
+{
+    "ConfigFiles": [
+      {
+        "destination": "/etc/loki/loki-local-config.yaml",
+        "group": "grafana",
+        "mode": "600",
+        "owner": "loki",
+        "service": "loki",
+        "source": "loki-local-config.pktpl.hcl"
+      }
+    ],
+    "Daemons": {
+      "Loki": {
+        "enabled": true,
+        "name": "loki"
+      }
+    },
+    "Packages": {
+      "loki": {
+        "action": "install",
+        "name": "loki"
+      },
+      "nodeExporter": {
+        "action": "install",
+        "name": "prometheus-node-exporter"
+      },
+      "promtail": {
+        "action": "install",
+        "name": "loki-promtail"
+      }
+    },
+    "Repositories": {
+      "AlpineTesting": {
+        "enabled": true,
+        "name": "testing",
+        "type": "apk",
+        "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing"
+      }
+    },
+    "Users": {
+      "loki": {
+        "group": "grafana",
+        "home": "/srv/loki",
+        "shell": "/bin/nologin",
+        "username": "loki"
+      }
+    },
+    "Vars": {
+      "AlertManagerURL": "http://localhost:9092",
+      "AuthEnabled": false,
+      "GRPCPort": "9095",
+      "Group": "grafana",
+      "HTTPPort": "3099",
+      "LogLevel": "error",
+      "ObjectStore": "filesystem",
+      "S2": {
+        "APIKey": "",
+        "APISecretKey": "",
+        "BucketName": "",
+        "URL": ""
+      },
+      "SharedStore": "filesystem",
+      "StorageRoot": "/var/loki",
+      "User": "loki"
+    }
+  }

data/templates/go-test-go.tpl

@@ -0,0 +1,13 @@
+###  Go template test ###
+{{ if .Vars.AuthEnabled  }}
+auth_enabled: true
+{{ else  }}
+auth_enabled: false
+{{ end  }}
+
+server:
+  http_listen_port: {{ .Vars.HTTPPort }}
+  grpc_listen_port: {{ .Vars.GRPCPort }}
+  log_level: {{ .Vars.LogLevel }}
+
+### End Go template test ### 

data/templates/go-test-hcl.pktpl.hcl

@@ -0,0 +1,12 @@
+### HCL2 Template test ###
+%{ if Vars.AuthEnabled ~}
+auth_enabled: true
+%{ else }
+auth_enabled: false
+%{ endif }
+
+server:
+  http_listen_port: ${Vars.HTTPPort}
+  grpc_listen_port: ${Vars.GRPCPort}
+  log_level: ${Vars.LogLevel}
+### END HCL Template test ###

data/templates/loki-local-config.tpl

@@ -1,13 +1,13 @@
-{{ if .AuthEnabled  }}
+{{ if .Vars.AuthEnabled  }}
 auth_enabled: true
 {{ else  }}
 auth_enabled: false
 {{ end  }}
 
 server:
-  http_listen_port: {{ .HTTPPort }}
-  grpc_listen_port: {{ .GRPCPort }}
-  log_level: {{ .LogLevel }}
+  http_listen_port: {{ .Vars.HTTPPort }}
+  grpc_listen_port: {{ .Vars.GRPCPort }}
+  log_level: {{ .Vars.LogLevel }}
 
 ingester:
   wal:

pkg/templater/files.go

@@ -5,16 +5,8 @@ import (
 	"path/filepath"
 	"strconv"
 
-	"bytes"
-	encjson "encoding/json"
 	"fmt"
 	"os"
-	"text/template"
-
-	"github.com/hashicorp/hcl/v2"
-	"github.com/hashicorp/hcl/v2/hclsyntax"
-	"github.com/zclconf/go-cty/cty"
-	ctyjson "github.com/zclconf/go-cty/cty/json"
 
 	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
 )
@@ -27,19 +19,20 @@ type ConfigFile struct {
 	Owner        string `json:"owner"`                          // The configuration file owner
 	Service      string `json:"service"`                        // Service to restart after configuration generation
 	Group        string `json:"group"`                          // The configuration file group owner
+	TemplateDir  string
 }
 
 // Generate the configuration file from the template (hcl or json)
 func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error {
 	var template string
+	cf.TemplateDir = templateDir
 	dest := filepath.Join(root, cf.Destination)
-	source := filepath.Join(templateDir, cf.Source)
 	intMod, err := strconv.ParseInt(cf.Mode, 8, 64)
 	if err != nil {
 		return (err)
 	}
 
-	template, err = cf.ProcessTemplate(source, values)
+	template, err = cf.ProcessTemplate(root, values)
 	if err != nil {
 		return fmt.Errorf("Process templates failed with error: %v", err)
 	}
@@ -57,97 +50,22 @@ func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) e
 }
 
 // Process the template with the provided values
-func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) {
+func (cf *ConfigFile) ProcessTemplate(root string, values []byte) (string, error) {
 	var result string
 	var err error
 
 	if cf.TemplateType == "hcl" {
 		// The template is an hcl template so we call processHCLTemplate
-		result, err = cf.processHCLTemplate(source, values)
+		result, err = utils.ProcessHCLTemplate(filepath.Join(cf.TemplateDir, cf.Source), values)
 		if err != nil {
 			return "", fmt.Errorf("Process HCL template failed with error: %v", err)
 		}
 	} else if cf.TemplateType == "go" {
 		// The template is a go template so we call processGoTemplate
-		result, err = cf.processGoTemplate(source, values)
+		result, err = utils.ProcessGoTemplate(filepath.Join(cf.TemplateDir, cf.Source), values)
 		if err != nil {
 			return "", fmt.Errorf("Process GO template failed with error: %v", err)
 		}
 	}
 	return result, nil
 }
-
-// The actual template processing for Go templates
-func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) {
-
-	// The JSON configuration
-	var confData map[string]interface{}
-	var res bytes.Buffer
-
-	err := encjson.Unmarshal(configValues, &confData)
-	utils.CheckErr(err)
-
-	// Read the template
-	data, err := os.ReadFile(file)
-	utils.CheckErr(err)
-
-	tpl, err := template.New("conf").Parse(string(data))
-	utils.CheckErr(err)
-
-	utils.CheckErr(tpl.Execute(&res, confData["Config"]))
-
-	return res.String(), nil
-}
-
-// The actual template processing for HCL templates
-func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) {
-
-	fct, err := os.ReadFile(file)
-	utils.CheckErr(err)
-
-	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
-	utils.CheckDiags(diags)
-
-	// Retrieve values from JSON
-	var varsVal cty.Value
-	ctyType, err := ctyjson.ImpliedType(config)
-	if err != nil {
-		return "", err
-		/* Maybe one day
-		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
-		if diags.HasErrors() {
-			panic(diags.Error())
-		}
-		varsVal, diags = cexpr.Value(&hcl.EvalContext{})
-		fmt.Println(cexpr.Variables())
-		checkDiags(diags)
-		*/
-	} else {
-		varsVal, err = ctyjson.Unmarshal(config, ctyType)
-		utils.CheckErr(err)
-	}
-
-	ctx := &hcl.EvalContext{
-		Variables: varsVal.AsValueMap(),
-	}
-
-	for n := range ctx.Variables {
-		if !hclsyntax.ValidIdentifier(n) {
-			return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
-		}
-	}
-
-	for _, traversal := range expr.Variables() {
-		root := traversal.RootName()
-		if _, ok := ctx.Variables[root]; !ok {
-			return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
-		}
-	}
-
-	val, diags := expr.Value(ctx)
-	if diags.HasErrors() {
-		return "", diags
-	}
-
-	return val.AsString(), nil
-}

pkg/templater/files_test.go

@@ -0,0 +1,72 @@
+package templater
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func TestProcessHCLTemplate(t *testing.T) {
+	// load the Full configuration from a file
+	values, err := ioutil.ReadFile("../../data/config/loki-stack.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	_, err = processHCLTemplate("../../data/templates/loki-local-config.tpl", values)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+}
+
+func TestProcessGoTemplate(t *testing.T) {
+	// load the Full configuration from a file
+	values, err := ioutil.ReadFile("../../data/config/loki-stack.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	_, err = processGoTemplate("../../data/templates/loki-local-config.tpl", values)
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestProcessTemplate(t *testing.T) {
+
+	goFile := ConfigFile{
+		Destination:  "/tmp/test.txt",
+		Source:       "loki-local-config.tpl",
+		TemplateType: "go",
+		Mode:         "700",
+		TemplateDir:  "../../data/templates/",
+	}
+
+	values, err := ioutil.ReadFile("../../data/config/loki-stack.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	_, err = goFile.ProcessTemplate("/tmp/", values)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+}
+
+/*
+func TestGenerate(t *testing.T) {
+
+	tFile := ConfigFile{
+		Destination:  "/tmp/test.txt",
+		Source:       "../../data/templates/loki-local-config.tpl",
+		TemplateType: "go",
+		Mode:         "700",
+	}
+
+	values := json.Unmarshal(File.)
+
+	_, err := tFile.Generate("/tmp/","../../data/templates/",values)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+}
+*/

pkg/templater/main.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os"
 
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
 	"github.com/imdario/mergo"
 )
 
@@ -80,10 +81,14 @@ func (tc *TemplaterConfig) New(confpath string, templateDir string, rootDir stri
 }
 
 // Process the services contained in the configuration "object"
-func (tc *TemplaterConfig) ManageServices() error {
+func (tc *TemplaterConfig) ManageServices(dryRun bool) error {
 	// Get global vars to add on each service
 	gbls := tc.GlobalService.Vars
 
+	if dryRun {
+		utils.DryRun = dryRun
+	}
+
 	for name, svr := range tc.Services {
 		err := mergo.Merge(&svr.Vars, gbls)
 		if err != nil {

pkg/templater/main_test.go

@@ -0,0 +1,17 @@
+package templater
+
+import "testing"
+
+func TestManageService(t *testing.T) {
+	var hostConfig TemplaterConfig
+
+	err := hostConfig.New("../../data/config/loki-stack.json", "../../data/templates/", "/tmp/testing")
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+
+	err = hostConfig.ManageServices(true)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+}

pkg/templater/repo-apk.go

@@ -78,7 +78,7 @@ func (hr *APKRepository) Update() error {
 
 // FIXME
 func (hr *APKRepository) Delete() error {
-	fileBytes, err := ioutil.ReadFile("/etc/apk/repositories")
+	fileBytes, err := ioutil.ReadFile(APKConfigFile)
 	if err != nil {
 		return err
 	}
@@ -90,13 +90,17 @@ func (hr *APKRepository) Delete() error {
 }
 
 func (hr *APKRepository) Manage() error {
-	if hr.Enabled {
-		if err := hr.Add(); err != nil {
-			return err
-		}
-		log.Println("\tUpdating apk repositories")
-		return hr.Update()
+	if utils.DryRun {
+		return nil
 	} else {
-		return hr.Delete()
+		if hr.Enabled {
+			if err := hr.Add(); err != nil {
+				return err
+			}
+			log.Println("\tUpdating apk repositories")
+			return hr.Update()
+		} else {
+			return hr.Delete()
+		}
 	}
 }

pkg/utils/main.go

@@ -2,11 +2,14 @@ package utils
 
 import (
 	"bytes"
+	"fmt"
 	"os/exec"
 
 	"github.com/hashicorp/hcl/v2"
 )
 
+var DryRun = false
+
 func CheckErr(e error) {
 	if e != nil {
 		panic(e)
@@ -21,12 +24,19 @@ func CheckDiags(diag hcl.Diagnostics) {
 
 // Execute a system command ...
 func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) {
-	var stdOut bytes.Buffer
-	var stdErr bytes.Buffer
+	if DryRun {
+		stdOut := []byte(fmt.Sprintf("CMD %s\n", name))
+		stdErr := []byte("STDERR\n")
 
-	cmd := exec.Command(name, arg...)
-	cmd.Stderr = &stdErr
-	cmd.Stdout = &stdOut
-	err := cmd.Run()
-	return stdOut.Bytes(), stdErr.Bytes(), err
+		return stdOut, stdErr, nil
+	} else {
+		var stdOut bytes.Buffer
+		var stdErr bytes.Buffer
+
+		cmd := exec.Command(name, arg...)
+		cmd.Stderr = &stdErr
+		cmd.Stdout = &stdOut
+		err := cmd.Run()
+		return stdOut.Bytes(), stdErr.Bytes(), err
+	}
 }

pkg/utils/templates.go

@@ -0,0 +1,89 @@
+package utils
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os"
+	"text/template"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	ctyjson "github.com/zclconf/go-cty/cty/json"
+)
+
+// The actual template processing for Go templates
+func ProcessGoTemplate(file string, configValues []byte) (string, error) {
+
+	// The JSON configuration
+	var confData map[string]interface{}
+	var res bytes.Buffer
+
+	err := json.Unmarshal(configValues, &confData)
+	CheckErr(err)
+
+	// Read the template
+	templateData, err := os.ReadFile(file)
+	CheckErr(err)
+
+	tpl, err := template.New("conf").Parse(string(templateData))
+	CheckErr(err)
+
+	CheckErr(tpl.Execute(&res, confData))
+
+	return res.String(), nil
+}
+
+// The actual template processing for HCL templates
+func ProcessHCLTemplate(file string, config []byte) (string, error) {
+
+	fct, err := os.ReadFile(file)
+	CheckErr(err)
+
+	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
+	CheckDiags(diags)
+
+	// Retrieve values from JSON
+	var varsVal cty.Value
+	ctyType, err := ctyjson.ImpliedType(config)
+	if err != nil {
+		return "", err
+		/* Maybe one day
+		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
+		if diags.HasErrors() {
+			panic(diags.Error())
+		}
+		varsVal, diags = cexpr.Value(&hcl.EvalContext{})
+		fmt.Println(cexpr.Variables())
+		checkDiags(diags)
+		*/
+	} else {
+		varsVal, err = ctyjson.Unmarshal(config, ctyType)
+		CheckErr(err)
+	}
+
+	ctx := &hcl.EvalContext{
+		Variables: varsVal.AsValueMap(),
+	}
+
+	for n := range ctx.Variables {
+		if !hclsyntax.ValidIdentifier(n) {
+			return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
+		}
+	}
+
+	for _, traversal := range expr.Variables() {
+		root := traversal.RootName()
+		if _, ok := ctx.Variables[root]; !ok {
+			return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
+		}
+	}
+
+	val, diags := expr.Value(ctx)
+	if diags.HasErrors() {
+		return "", diags
+	}
+
+	return val.AsString(), nil
+}

pkg/utils/templates_test.go

@@ -0,0 +1,34 @@
+package utils
+
+import (
+	"io/ioutil"
+	"testing"
+)
+
+func TestProcessHCLTemplate(t *testing.T) {
+	// load the Full configuration from a file
+	values, err := ioutil.ReadFile("../../data/config/go-test-conf.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	data, err := ProcessHCLTemplate("../../data/templates/go-test-hcl.pktpl.hcl", values)
+	if err != nil {
+		t.Errorf(err.Error())
+	}
+	t.Logf("%s", data)
+}
+
+func TestProcessGoTemplate(t *testing.T) {
+	// load values from testing json file
+	values, err := ioutil.ReadFile("../../data/config/go-test-conf.json")
+	if err != nil {
+		t.Error(err)
+	}
+
+	data, err := ProcessGoTemplate("../../data/templates/go-test-go.tpl", values)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("%s", data)
+}