Complete rewrite of the bootstraper tool

.gitignore

@@ -14,4 +14,4 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
-
+bin/

Makefile

@@ -2,12 +2,14 @@ LINT_ARGS ?= ./...
 DESTDIR ?= "/usr/local"
 
 bin:
-	GOOS=linux go build -o bin/templater-linux cmd/templater.go
+	GOOS=linux go build -o bin/templater-linux cmd/templater/main.go
+	GOOS=linux go build -o bin/bootstraper-linux cmd/bootstraper/main.go
 	upx bin/templater-linux
 	upx bin/templaster-server
 
 install:
 	cp bin/templater-linux $(DESTDIR)/bin/templater
+	cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper
 
 uninstall:
 	rm $(DESTDIR)/bin/templater

cmd/bootstraper/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
+	"github.com/alexflint/go-arg"
+)
+
+func main() {
+
+	var args struct {
+		Config            string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"`
+		TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"`
+	}
+
+	arg.MustParse(&args)
+
+	var hostConfig templater.TemplaterConfig
+
+	err := hostConfig.New(args.Config, args.TemplateDirectory)
+	if err != nil {
+		panic(err)
+	}
+	if err = hostConfig.ManageServices(); err != nil {
+		panic(err)
+	}
+}

cmd/templater/main.go

@@ -55,13 +55,13 @@ func main() {
 			config = []byte(args.Config)
 		}
 
-		result := ""
+		var file templater.ConfigFile
-		if templateType == "go" {
+		file.Source = templateFile
-			result = templater.ProcessGoTemplate(templateFile, config)
+		file.TemplateType = templateType
-		} else if templateType == "hcl" {
+
-			result = templater.ProcessHCLTemplate(templateFile, config)
+		result, err := file.ProcessTemplate(templateFile, config)
-		} else {
+		if err != nil {
-			panic(fmt.Errorf("Unsupported template type"))
+			panic(err)
 		}
 		if output == "stdout" {
 			fmt.Printf("%s", result)

data/config/test-services.json

@@ -0,0 +1,92 @@
+{
+  "Name": "LokiStack",
+  "Global": {
+    "Vars": {
+      "EnableLoki": true,
+      "EnableGrafana": false,
+      "EnablePrometheus" : false
+    },
+    "ConfigFiles": [
+      {
+        "destination": "/etc/hosts",
+        "source": "hosts.pktpl.hcl",
+        "mode": "600",
+        "owner": "root",
+        "group": "root"
+      }
+    ]
+  },
+  "Services": {
+    "Loki": {
+      "EnabledBy": { "var": "EnableLoki", "value": true },
+      "ConfigFiles": [
+        {
+          "destination": "/etc/loki/loki-local-config.yaml",
+          "source": "loki-local-config.pktpl.hcl",
+          "mode": "600",
+          "owner": "loki",
+          "group": "grafana"
+        }
+      ],
+      "Vars": {
+        "AuthEnabled": false,
+        "User": "loki",
+        "Group": "grafana",
+        "HTTPPort": "3100",
+        "GRPCPort": "9096",
+        "AlertManagerURL": "http://localhost:9093",
+        "StorageRoot": "/var/loki",
+        "SharedStore": "filesystem",
+        "ObjectStore": "filesystem",
+        "LogLevel": "error",
+        "S3": {
+            "URL": "",
+            "BucketName": "",
+            "APIKey": "",
+            "APISecretKey": ""
+        }
+      },
+      "Daemon": {
+        "name": "loki",
+        "enabled": true
+      },
+      "Users": {
+        "loki": {
+          "username": "loki",
+          "group": "grafana",
+          "home" : "/srv/loki",
+          "shell": "/bin/nologin"
+        }
+      }
+    },
+    "Grafana": {
+      "EnabledBy": { "var": "EnableGrafana", "value": true },
+      "ConfigFiles": [
+        {
+          "destination": "/etc/grafana.ini",
+          "source": "grafana.ini.pktpl.hcl",
+          "mode": "700",
+          "owner": "grafana",
+          "group": "grafana"
+        }
+      ],
+      "Vars": {
+        "AuthEnabled": false,
+        "User": "toto",
+        "Group": "grafana"
+      },
+      "Users": {
+        "grafana": {
+          "username": "grafana",
+          "group": "grafana",
+          "home": "/srv/grafana",
+          "shell": "/bin/nologin"
+        }
+      },
+      "Daemon": {
+        "name": "grafana",
+        "enabled": true
+      }
+    }
+  }
+}

data/config/test-servies-2.json

@@ -0,0 +1,61 @@
+{
+  "Name": "OpenNebula",
+  "Services": {
+    "Global": {
+      "Vars": {
+        "EnableOpenNebula": true
+      },
+      "ConfigFiles": [
+        {
+          "destination": "/etc/hosts",
+          "source": "hosts.pktpl.hcl",
+          "mode": "600",
+          "owner": "root",
+          "group": "root"
+        }
+      ]
+    },
+    "OpenNebula": {
+      "EnabledBy": { "var": "EnableOpenNebula", "value": true },
+      "ConfigFiles": [
+        {
+          "destination": "/etc/one/oned.conf",
+          "source": "loki-local-config.pktpl.hcl",
+          "mode": "600",
+          "owner": "loki",
+          "group": "grafana"
+        }
+      ],
+      "Vars": {
+        "AuthEnabled": false,
+        "User": "loki",
+        "Group": "grafana",
+        "HTTPPort": "3100",
+        "GRPCPort": "9096",
+        "AlertManagerURL": "http://localhost:9093",
+        "StorageRoot": "/var/loki",
+        "SharedStore": "filesystem",
+        "ObjectStore": "filesystem",
+        "LogLevel": "error",
+        "S3": {
+            "URL": "",
+            "BucketName": "",
+            "APIKey": "",
+            "APISecretKey": ""
+        }
+      },
+      "Daemon": {
+        "name": "oned",
+        "enabled": true
+      },
+      "Users": {
+        "loki": {
+          "username": "oneadmin",
+          "group": "oneadmin",
+          "home" : "/var/lib/one",
+          "shell": "/bin/sh"
+        }
+      }
+    }
+  }
+}

data/config/test.json

@@ -1,26 +0,0 @@
-{
-  "Name": "loki",
-  "ConfigFiles": [
-    {
-      "destination": "/etc/loki/loki-local-config.yaml",
-      "source": "loki-local-config.pktpl.hcl",
-      "mod": "600"
-    }
-  ],
-  "AuthEnabled": false,
-  "User": "loki",
-  "Group": "grafana",
-  "HTTPPort": "3100",
-  "GRPCPort": "9096",
-  "AlertManagerURL": "http://localhost:9093",
-  "StorageRoot": "/var/loki",
-  "SharedStore": "filesystem",
-  "ObjectStore": "filesystem",
-  "LogLevel": "error",
-  "S3": {
-      "URL": "",
-      "BucketName": "",
-      "APIKey": "",
-      "APISecretKey": ""
-  }
-}

data/templates/grafana.ini.pktpl.hcl

@@ -0,0 +1,5 @@
+%{ if AuthEnabled ~}
+auth_enabled: true
+%{ else }
+auth_enabled: false
+%{ endif }

pkg/templater/files.go

@@ -0,0 +1,151 @@
+package templater
+
+import (
+	"path/filepath"
+	"strconv"
+
+	"bytes"
+	encjson "encoding/json"
+	"fmt"
+	"os"
+	"text/template"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	ctyjson "github.com/zclconf/go-cty/cty/json"
+
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
+)
+
+type ConfigFile struct {
+	Destination  string `form:"destination" json:"destination"` // Where do we write the configuration file
+	Source       string `form:"source" json:"source"`           // The template file short name
+	TemplateType string `json:"type"`                           // The template file type (hcl or gotemplate)
+	Mode         string `form:"mod" json:"mode"`                // The configuration file final permissions (mode)
+	Owner        string `json:"owner"`                          // The configuration file owner
+	Group        string `json:"group"`                          // The configuration file group owner
+}
+
+// Generate the configuration file from the template (hcl or json)
+func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error {
+	var template string
+	dest := filepath.Join(root, cf.Destination)
+	source := filepath.Join(templateDir, cf.Source)
+	intMod, err := strconv.ParseInt(cf.Mode, 8, 64)
+	if err != nil {
+		return (err)
+	}
+
+	template, err = cf.ProcessTemplate(source, values)
+	if err != nil {
+		return fmt.Errorf("Process templates failed with error: %v", err)
+	}
+	dirname := filepath.Dir(dest)
+	err = os.MkdirAll(dirname, os.FileMode(int(0700)))
+	if err != nil {
+		return fmt.Errorf("Process templates failed with error: %v", err)
+	}
+	err = os.WriteFile(dest, []byte(template), os.FileMode(intMod))
+	if err != nil {
+		return fmt.Errorf("Process templates failed with error: %v", err)
+	}
+	return nil
+}
+
+// Process the template with the provided values
+func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) {
+	var result string
+	var err error
+
+	fmt.Printf("Processing %s\n", source)
+	if cf.TemplateType == "hcl" {
+		// The template is an hcl template so we call processHCLTemplate
+		result, err = cf.processHCLTemplate(source, values)
+		if err != nil {
+			return "", fmt.Errorf("Process HCL template failed with error: %v", err)
+		}
+	} else if cf.TemplateType == "go" {
+		// The template is a go template so we call processGoTemplate
+		result, err = cf.processGoTemplate(source, values)
+		if err != nil {
+			return "", fmt.Errorf("Process GO template failed with error: %v", err)
+		}
+	}
+	return result, nil
+}
+
+// The actual template processing for Go templates
+func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) {
+
+	// The JSON configuration
+	var confData map[string]interface{}
+	var res bytes.Buffer
+
+	err := encjson.Unmarshal(configValues, &confData)
+	utils.CheckErr(err)
+
+	// Read the template
+	data, err := os.ReadFile(file)
+	utils.CheckErr(err)
+
+	tpl, err := template.New("conf").Parse(string(data))
+	utils.CheckErr(err)
+
+	utils.CheckErr(tpl.Execute(&res, confData["Config"]))
+
+	return res.String(), nil
+}
+
+// The actual template processing for HCL templates
+func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) {
+
+	fct, err := os.ReadFile(file)
+	utils.CheckErr(err)
+
+	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
+	utils.CheckDiags(diags)
+
+	// Retrieve values from JSON
+	var varsVal cty.Value
+	ctyType, err := ctyjson.ImpliedType(config)
+	if err != nil {
+		return "", err
+		/* Maybe one day
+		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
+		if diags.HasErrors() {
+			panic(diags.Error())
+		}
+		varsVal, diags = cexpr.Value(&hcl.EvalContext{})
+		fmt.Println(cexpr.Variables())
+		checkDiags(diags)
+		*/
+	} else {
+		varsVal, err = ctyjson.Unmarshal(config, ctyType)
+		utils.CheckErr(err)
+	}
+
+	ctx := &hcl.EvalContext{
+		Variables: varsVal.AsValueMap(),
+	}
+
+	for n := range ctx.Variables {
+		if !hclsyntax.ValidIdentifier(n) {
+			panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n))
+		}
+	}
+
+	for _, traversal := range expr.Variables() {
+		root := traversal.RootName()
+		if _, ok := ctx.Variables[root]; !ok {
+			panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()))
+		}
+	}
+
+	val, diags := expr.Value(ctx)
+	if diags.HasErrors() {
+		panic(diags.Error())
+	}
+
+	return val.AsString(), nil
+}

pkg/templater/main.go

@@ -1,89 +1,90 @@
 package templater
 
 import (
-	"bytes"
-	encjson "encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
-	"text/template"
 
-	"github.com/hashicorp/hcl/v2"
+	"github.com/imdario/mergo"
-	"github.com/hashicorp/hcl/v2/hclsyntax"
-	"github.com/zclconf/go-cty/cty"
-	ctyjson "github.com/zclconf/go-cty/cty/json"
-
-	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
 )
 
-func ProcessGoTemplate(file string, config []byte) string {
+var CacheFilePath = "/var/cache/templater.db"
 
-	// The JSON configuration
+type SimpleCondition struct {
-	var confData map[string]interface{}
+	Var   string `json:"var"`
-	var res bytes.Buffer
+	Value bool   `json:"value"`
-
-	err := encjson.Unmarshal(config, &confData)
-	utils.CheckErr(err)
-
-	// Read the template
-	data, err := os.ReadFile(file)
-	utils.CheckErr(err)
-
-	tpl, err := template.New("conf").Parse(string(data))
-	utils.CheckErr(err)
-
-	utils.CheckErr(tpl.Execute(&res, confData))
-
-	return res.String()
 }
 
-func ProcessHCLTemplate(file string, config []byte) string {
+type TemplaterConfig struct {
+	Name              string             `json:"Name"`
+	TemplateDirectory string             `json:"TemplateDirectory"`
+	Services          map[string]Service `json:"Services"`
+	GlobalService     Service            `json:"Global"`
+}
 
-	fct, err := os.ReadFile(file)
+func (tc *TemplaterConfig) loadCache() error {
-	utils.CheckErr(err)
+	// Load globals from cache
-
+	var cache Service
-	expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
+	err := Load(CacheFilePath, &cache)
-	utils.CheckDiags(diags)
-
-	// Retrieve values from JSON
-	var varsVal cty.Value
-	ctyType, err := ctyjson.ImpliedType(config)
 	if err != nil {
-		panic(err)
+		fmt.Printf("Warning: No globals to load\n")
-		/* Maybe one day
+	}
-		cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
+
-		if diags.HasErrors() {
+	err = mergo.Merge(&tc.GlobalService, cache)
-			panic(diags.Error())
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (tc *TemplaterConfig) New(confpath string, templateDir string) error {
+	// Load stored globals if needed
+	lerr := tc.loadCache()
+	if lerr != nil {
+		return lerr
+	}
+	// Check if the configuration path is a Directory or a file
+	fileInfo, err := os.Stat(confpath)
+	if err != nil {
+		return err
+	}
+
+	if fileInfo.IsDir() {
+		// The conf path is a directory we load all the files and merge data
+		files, err := ioutil.ReadDir(confpath)
+		if err != nil {
+			return fmt.Errorf("Templater configuration load failed with error: %v", err)
+		}
+		for _, file := range files {
+			fname := fmt.Sprintf("%s/%s", confpath, file.Name())
+			var ntc TemplaterConfig
+			err := Load(fname, &ntc)
+			if err != nil {
+				return fmt.Errorf("Templater configuration load failed with error: %v", err)
+			}
+			err = mergo.Merge(tc, ntc)
+			if err != nil {
+				return fmt.Errorf("Templater configuration load failed with error: %v", err)
+			}
 		}
-		varsVal, diags = cexpr.Value(&hcl.EvalContext{})
-		fmt.Println(cexpr.Variables())
-		checkDiags(diags)
-		*/
 	} else {
-		varsVal, err = ctyjson.Unmarshal(config, ctyType)
+		// The conf path is a file we only load this file (of course)
-		utils.CheckErr(err)
+		err = Load(confpath, tc)
-	}
+		if err != nil {
-
+			return fmt.Errorf("Confiuration read failed with error: %v", err)
-	ctx := &hcl.EvalContext{
-		Variables: varsVal.AsValueMap(),
-	}
-
-	for n := range ctx.Variables {
-		if !hclsyntax.ValidIdentifier(n) {
-			panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n))
 		}
 	}
 
-	for _, traversal := range expr.Variables() {
+	tc.TemplateDirectory = templateDir
-		root := traversal.RootName()
+
-		if _, ok := ctx.Variables[root]; !ok {
+	return nil
-			panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()))
-		}
 }
 
-	val, diags := expr.Value(ctx)
+func (tc *TemplaterConfig) ManageServices() error {
-	if diags.HasErrors() {
+	for _, svr := range tc.Services {
-		panic(diags.Error())
+		if err := svr.Manage(tc.TemplateDirectory); err != nil {
+			return err
 		}
-
+	}
-	return val.AsString()
+	return nil
 }

pkg/templater/persist.go

@@ -0,0 +1,51 @@
+package templater
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"sync"
+)
+
+var lock sync.Mutex
+
+func marshal(v interface{}) (io.Reader, error) {
+	b, err := json.MarshalIndent(v, "", "\t")
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewReader(b), nil
+}
+
+func unmarshal(r io.Reader, v interface{}) error {
+	return json.NewDecoder(r).Decode(v)
+}
+
+func Save(path string, v interface{}) error {
+	lock.Lock()
+	defer lock.Unlock()
+	f, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("Saving Templater configuration failed with error : %v", err)
+	}
+	defer f.Close()
+	r, err := marshal(v)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(f, r)
+	return err
+}
+
+func Load(path string, v interface{}) error {
+	lock.Lock()
+	defer lock.Unlock()
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return unmarshal(f, v)
+}

pkg/templater/services.go

@@ -0,0 +1,51 @@
+package templater
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+)
+
+type Service struct {
+	EnabledBy   SimpleCondition        `json:"EnabledBy"`
+	ConfigFiles []ConfigFile           `json:"ConfigFiles"`
+	Vars        map[string]interface{} `json:"Vars"`
+	Daemon      SystemService          `json:"Daemon"`
+	Users       map[string]SystemUser  `json:"Users"`
+}
+
+func (s *Service) Manage(templateDir string) error {
+	err := processConfigFiles(s.ConfigFiles, s.Vars, templateDir)
+	if err != nil {
+		return fmt.Errorf("ProcessingTemplatesFailed with error: %v", err)
+	}
+
+	err = s.Daemon.Manage()
+	if err != nil {
+		return fmt.Errorf("Error managing service daemons: %v", err)
+	}
+	return nil
+}
+
+func processConfigFiles(tpls []ConfigFile, variables map[string]interface{}, templateDir string) error {
+	values, err := json.Marshal(variables)
+	if err != nil {
+		return fmt.Errorf("Error unmarshaling values on template process; %v", err)
+	}
+
+	for _, tpl := range tpls {
+		fileExt := filepath.Ext(tpl.Source)
+		if fileExt == ".hcl" {
+			tpl.TemplateType = "hcl"
+		} else if fileExt == ".tpl" {
+			tpl.TemplateType = "go"
+		} else {
+			return fmt.Errorf("Unsupported file extention %s, templates extensions have to be '.hcl' or '.tpl'", fileExt)
+		}
+		if err := tpl.Generate("/tmp/test", templateDir, values); err != nil {
+			return fmt.Errorf("Template %s generation failed with error %v", tpl.Source, err)
+		}
+
+	}
+	return nil
+}

pkg/templater/system_services.go

@@ -0,0 +1,74 @@
+package templater
+
+import (
+	"fmt"
+	"os"
+
+	"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
+)
+
+type SystemService struct {
+	Name    string `json:"name"`
+	Enabled bool   `json:"enabled"`
+	Type    string `json:"type"`
+}
+
+func (sys *SystemService) SetType() {
+	systemdRunDirectory := "/run/systemd/system"
+	openRcBinaryFile := "/sbin/openrc"
+
+	// Check if the configuration path is a Directory or a file
+	fileInfo, err := os.Stat(systemdRunDirectory)
+	if err == nil {
+		if fileInfo.IsDir() {
+			sys.Type = "systemd"
+		}
+	}
+
+	fileInfo, err = os.Stat(openRcBinaryFile)
+	if err == nil {
+		if fileInfo.IsDir() {
+			return
+		}
+		sys.Type = "openrc"
+	}
+}
+
+func (sys *SystemService) Manage() error {
+	if sys.Type == "" {
+		sys.SetType()
+	}
+	if sys.Enabled {
+		err := sys.Enable()
+		if err != nil {
+			return err
+		}
+	} else {
+		fmt.Printf("Nothing to do for daemon %s\n", sys.Name)
+	}
+	return nil
+}
+
+func (sys *SystemService) Start() error {
+	fmt.Printf("Starting %s\n", sys.Name)
+	return nil
+}
+
+func (sys *SystemService) Stop() error {
+	fmt.Printf("Stoping %s\n", sys.Name)
+	return nil
+}
+
+func (sys *SystemService) Enable() error {
+	if sys.Type == "systemd" {
+		_, stdErr, err := utils.RunSystemCommand("systemctl", "enable", sys.Name)
+		if err != nil {
+			return fmt.Errorf("System service %s \n * Enable error:\n    - %s", sys.Name, stdErr)
+		}
+	} else if sys.Type == "openrc" {
+		return nil
+	} else {
+		return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name)
+	}
+	return nil
+}

pkg/templater/system_users.go

@@ -0,0 +1,8 @@
+package templater
+
+type SystemUser struct {
+	UserName string `json:"username"`
+	Group    string `json:"group"`
+	Home     string `json:"home"`
+	Shell    string `json:"shell"`
+}

pkg/utils/main.go

@@ -1,6 +1,9 @@
 package utils
 
 import (
+	"bytes"
+	"os/exec"
+
 	"github.com/hashicorp/hcl/v2"
 )
 
@@ -15,3 +18,15 @@ func CheckDiags(diag hcl.Diagnostics) {
 		panic(diag.Error())
 	}
 }
+
+// Execute a system command ...
+func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) {
+	var stdOut bytes.Buffer
+	var stdErr bytes.Buffer
+
+	cmd := exec.Command(name, arg...)
+	cmd.Stderr = &stdErr
+	cmd.Stdout = &stdOut
+	err := cmd.Run()
+	return stdOut.Bytes(), stdErr.Bytes(), err
+}