Compare commits

...

2 Commits

Author SHA1 Message Date
Philippe Caseiro a9aa10cb3e feat(test): adding tests for the "File" struct
feat(test): moving util funcs into utils module
2022-07-06 14:42:26 +02:00
Philippe Caseiro d9379f7e33 feat(test): adding unit tests 2022-07-06 14:37:00 +02:00
13 changed files with 306 additions and 109 deletions

View File

@ -11,6 +11,7 @@ func main() {
Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"`
TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"`
RootDirectory string `arg:"-r,--root-dir,env:ROOT_DIR" help:"Generate files with this root instead of /" default:"/"`
DryRun bool `arg:"-d,--dry-run,env:DRY_RUN" help:"Dry run do not really complete actions" default:"false"`
}
arg.MustParse(&args)
@ -21,7 +22,7 @@ func main() {
if err != nil {
panic(err)
}
if err = hostConfig.ManageServices(); err != nil {
if err = hostConfig.ManageServices(args.DryRun); err != nil {
panic(err)
}
}

View File

@ -0,0 +1,66 @@
{
"ConfigFiles": [
{
"destination": "/etc/loki/loki-local-config.yaml",
"group": "grafana",
"mode": "600",
"owner": "loki",
"service": "loki",
"source": "loki-local-config.pktpl.hcl"
}
],
"Daemons": {
"Loki": {
"enabled": true,
"name": "loki"
}
},
"Packages": {
"loki": {
"action": "install",
"name": "loki"
},
"nodeExporter": {
"action": "install",
"name": "prometheus-node-exporter"
},
"promtail": {
"action": "install",
"name": "loki-promtail"
}
},
"Repositories": {
"AlpineTesting": {
"enabled": true,
"name": "testing",
"type": "apk",
"url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing"
}
},
"Users": {
"loki": {
"group": "grafana",
"home": "/srv/loki",
"shell": "/bin/nologin",
"username": "loki"
}
},
"Vars": {
"AlertManagerURL": "http://localhost:9092",
"AuthEnabled": false,
"GRPCPort": "9095",
"Group": "grafana",
"HTTPPort": "3099",
"LogLevel": "error",
"ObjectStore": "filesystem",
"S2": {
"APIKey": "",
"APISecretKey": "",
"BucketName": "",
"URL": ""
},
"SharedStore": "filesystem",
"StorageRoot": "/var/loki",
"User": "loki"
}
}

View File

@ -0,0 +1,13 @@
### Go template test ###
{{ if .Vars.AuthEnabled }}
auth_enabled: true
{{ else }}
auth_enabled: false
{{ end }}
server:
http_listen_port: {{ .Vars.HTTPPort }}
grpc_listen_port: {{ .Vars.GRPCPort }}
log_level: {{ .Vars.LogLevel }}
### End Go template test ###

View File

@ -0,0 +1,12 @@
### HCL2 Template test ###
%{ if Vars.AuthEnabled ~}
auth_enabled: true
%{ else }
auth_enabled: false
%{ endif }
server:
http_listen_port: ${Vars.HTTPPort}
grpc_listen_port: ${Vars.GRPCPort}
log_level: ${Vars.LogLevel}
### END HCL Template test ###

View File

@ -1,13 +1,13 @@
{{ if .AuthEnabled }}
{{ if .Vars.AuthEnabled }}
auth_enabled: true
{{ else }}
auth_enabled: false
{{ end }}
server:
http_listen_port: {{ .HTTPPort }}
grpc_listen_port: {{ .GRPCPort }}
log_level: {{ .LogLevel }}
http_listen_port: {{ .Vars.HTTPPort }}
grpc_listen_port: {{ .Vars.GRPCPort }}
log_level: {{ .Vars.LogLevel }}
ingester:
wal:

View File

@ -5,16 +5,8 @@ import (
"path/filepath"
"strconv"
"bytes"
encjson "encoding/json"
"fmt"
"os"
"text/template"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
ctyjson "github.com/zclconf/go-cty/cty/json"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
)
@ -27,19 +19,20 @@ type ConfigFile struct {
Owner string `json:"owner"` // The configuration file owner
Service string `json:"service"` // Service to restart after configuration generation
Group string `json:"group"` // The configuration file group owner
TemplateDir string
}
// Generate the configuration file from the template (hcl or json)
func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error {
var template string
cf.TemplateDir = templateDir
dest := filepath.Join(root, cf.Destination)
source := filepath.Join(templateDir, cf.Source)
intMod, err := strconv.ParseInt(cf.Mode, 8, 64)
if err != nil {
return (err)
}
template, err = cf.ProcessTemplate(source, values)
template, err = cf.ProcessTemplate(root, values)
if err != nil {
return fmt.Errorf("Process templates failed with error: %v", err)
}
@ -57,97 +50,22 @@ func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) e
}
// Process the template with the provided values
func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) {
func (cf *ConfigFile) ProcessTemplate(root string, values []byte) (string, error) {
var result string
var err error
if cf.TemplateType == "hcl" {
// The template is an hcl template so we call processHCLTemplate
result, err = cf.processHCLTemplate(source, values)
result, err = utils.ProcessHCLTemplate(filepath.Join(cf.TemplateDir, cf.Source), values)
if err != nil {
return "", fmt.Errorf("Process HCL template failed with error: %v", err)
}
} else if cf.TemplateType == "go" {
// The template is a go template so we call processGoTemplate
result, err = cf.processGoTemplate(source, values)
result, err = utils.ProcessGoTemplate(filepath.Join(cf.TemplateDir, cf.Source), values)
if err != nil {
return "", fmt.Errorf("Process GO template failed with error: %v", err)
}
}
return result, nil
}
// The actual template processing for Go templates
func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) {
// The JSON configuration
var confData map[string]interface{}
var res bytes.Buffer
err := encjson.Unmarshal(configValues, &confData)
utils.CheckErr(err)
// Read the template
data, err := os.ReadFile(file)
utils.CheckErr(err)
tpl, err := template.New("conf").Parse(string(data))
utils.CheckErr(err)
utils.CheckErr(tpl.Execute(&res, confData["Config"]))
return res.String(), nil
}
// The actual template processing for HCL templates
func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) {
fct, err := os.ReadFile(file)
utils.CheckErr(err)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
utils.CheckDiags(diags)
// Retrieve values from JSON
var varsVal cty.Value
ctyType, err := ctyjson.ImpliedType(config)
if err != nil {
return "", err
/* Maybe one day
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
if diags.HasErrors() {
panic(diags.Error())
}
varsVal, diags = cexpr.Value(&hcl.EvalContext{})
fmt.Println(cexpr.Variables())
checkDiags(diags)
*/
} else {
varsVal, err = ctyjson.Unmarshal(config, ctyType)
utils.CheckErr(err)
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
}
}
val, diags := expr.Value(ctx)
if diags.HasErrors() {
return "", diags
}
return val.AsString(), nil
}

View File

@ -0,0 +1,28 @@
package templater
import (
"io/ioutil"
"testing"
)
func TestProcessTemplate(t *testing.T) {
goFile := ConfigFile{
Destination: "/loki-config.test",
Source: "loki-local-config.tpl",
TemplateType: "go",
Mode: "700",
TemplateDir: "../../data/templates/",
}
values, err := ioutil.ReadFile("../../data/config/loki-stack.json")
if err != nil {
t.Error(err)
}
data, err := goFile.ProcessTemplate("/tmp/", values)
if err != nil {
t.Errorf(err.Error())
}
t.Log(data)
}

View File

@ -6,6 +6,7 @@ import (
"log"
"os"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
"github.com/imdario/mergo"
)
@ -80,10 +81,14 @@ func (tc *TemplaterConfig) New(confpath string, templateDir string, rootDir stri
}
// Process the services contained in the configuration "object"
func (tc *TemplaterConfig) ManageServices() error {
func (tc *TemplaterConfig) ManageServices(dryRun bool) error {
// Get global vars to add on each service
gbls := tc.GlobalService.Vars
if dryRun {
utils.DryRun = dryRun
}
for name, svr := range tc.Services {
err := mergo.Merge(&svr.Vars, gbls)
if err != nil {

View File

@ -0,0 +1,17 @@
package templater
import "testing"
func TestManageService(t *testing.T) {
var hostConfig TemplaterConfig
err := hostConfig.New("../../data/config/loki-stack.json", "../../data/templates/", "/tmp/testing")
if err != nil {
t.Errorf(err.Error())
}
err = hostConfig.ManageServices(true)
if err != nil {
t.Errorf(err.Error())
}
}

View File

@ -78,7 +78,7 @@ func (hr *APKRepository) Update() error {
// FIXME
func (hr *APKRepository) Delete() error {
fileBytes, err := ioutil.ReadFile("/etc/apk/repositories")
fileBytes, err := ioutil.ReadFile(APKConfigFile)
if err != nil {
return err
}
@ -90,13 +90,17 @@ func (hr *APKRepository) Delete() error {
}
func (hr *APKRepository) Manage() error {
if hr.Enabled {
if err := hr.Add(); err != nil {
return err
}
log.Println("\tUpdating apk repositories")
return hr.Update()
if utils.DryRun {
return nil
} else {
return hr.Delete()
if hr.Enabled {
if err := hr.Add(); err != nil {
return err
}
log.Println("\tUpdating apk repositories")
return hr.Update()
} else {
return hr.Delete()
}
}
}

View File

@ -2,11 +2,14 @@ package utils
import (
"bytes"
"fmt"
"os/exec"
"github.com/hashicorp/hcl/v2"
)
var DryRun = false
func CheckErr(e error) {
if e != nil {
panic(e)
@ -21,12 +24,19 @@ func CheckDiags(diag hcl.Diagnostics) {
// Execute a system command ...
func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) {
var stdOut bytes.Buffer
var stdErr bytes.Buffer
if DryRun {
stdOut := []byte(fmt.Sprintf("CMD %s\n", name))
stdErr := []byte("STDERR\n")
cmd := exec.Command(name, arg...)
cmd.Stderr = &stdErr
cmd.Stdout = &stdOut
err := cmd.Run()
return stdOut.Bytes(), stdErr.Bytes(), err
return stdOut, stdErr, nil
} else {
var stdOut bytes.Buffer
var stdErr bytes.Buffer
cmd := exec.Command(name, arg...)
cmd.Stderr = &stdErr
cmd.Stdout = &stdOut
err := cmd.Run()
return stdOut.Bytes(), stdErr.Bytes(), err
}
}

89
pkg/utils/templates.go Normal file
View File

@ -0,0 +1,89 @@
package utils
import (
"bytes"
"encoding/json"
"fmt"
"os"
"text/template"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
ctyjson "github.com/zclconf/go-cty/cty/json"
)
// The actual template processing for Go templates
func ProcessGoTemplate(file string, configValues []byte) (string, error) {
// The JSON configuration
var confData map[string]interface{}
var res bytes.Buffer
err := json.Unmarshal(configValues, &confData)
CheckErr(err)
// Read the template
templateData, err := os.ReadFile(file)
CheckErr(err)
tpl, err := template.New("conf").Parse(string(templateData))
CheckErr(err)
CheckErr(tpl.Execute(&res, confData))
return res.String(), nil
}
// The actual template processing for HCL templates
func ProcessHCLTemplate(file string, config []byte) (string, error) {
fct, err := os.ReadFile(file)
CheckErr(err)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
CheckDiags(diags)
// Retrieve values from JSON
var varsVal cty.Value
ctyType, err := ctyjson.ImpliedType(config)
if err != nil {
return "", err
/* Maybe one day
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
if diags.HasErrors() {
panic(diags.Error())
}
varsVal, diags = cexpr.Value(&hcl.EvalContext{})
fmt.Println(cexpr.Variables())
checkDiags(diags)
*/
} else {
varsVal, err = ctyjson.Unmarshal(config, ctyType)
CheckErr(err)
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
}
}
val, diags := expr.Value(ctx)
if diags.HasErrors() {
return "", diags
}
return val.AsString(), nil
}

View File

@ -0,0 +1,34 @@
package utils
import (
"io/ioutil"
"testing"
)
func TestProcessHCLTemplate(t *testing.T) {
// load the Full configuration from a file
values, err := ioutil.ReadFile("../../data/config/go-test-conf.json")
if err != nil {
t.Error(err)
}
data, err := ProcessHCLTemplate("../../data/templates/go-test-hcl.pktpl.hcl", values)
if err != nil {
t.Errorf(err.Error())
}
t.Logf("%s", data)
}
func TestProcessGoTemplate(t *testing.T) {
// load values from testing json file
values, err := ioutil.ReadFile("../../data/config/go-test-conf.json")
if err != nil {
t.Error(err)
}
data, err := ProcessGoTemplate("../../data/templates/go-test-go.tpl", values)
if err != nil {
t.Error(err)
}
t.Logf("%s", data)
}