From d428115f50289db33ac6a8c7db61eb7934d32d3f Mon Sep 17 00:00:00 2001 From: Philippe Caseiro Date: Wed, 15 Jun 2022 17:29:45 +0200 Subject: [PATCH] Complete rewrite of the bootstraper tool --- Makefile | 4 +- cmd/bootstraper/main.go | 26 +++++ cmd/{templater.go => templater/main.go} | 14 +-- data/config/test-services.json | 92 +++++++++++++++ data/config/test-servies-2.json | 61 ++++++++++ data/config/test.json | 26 ----- data/templates/grafana.ini.pktpl.hcl | 5 + data/templates/hosts.pktpl.hcl | 0 pkg/templater/files.go | 145 ++++++++++++++++++++++++ pkg/templater/main.go | 145 ++++++++++++------------ pkg/templater/persist.go | 51 +++++++++ pkg/templater/services.go | 51 +++++++++ pkg/templater/system_services.go | 74 ++++++++++++ pkg/templater/system_users.go | 8 ++ pkg/utils/main.go | 15 +++ 15 files changed, 611 insertions(+), 106 deletions(-) create mode 100644 cmd/bootstraper/main.go rename cmd/{templater.go => templater/main.go} (86%) create mode 100644 data/config/test-services.json create mode 100644 data/config/test-servies-2.json delete mode 100644 data/config/test.json create mode 100644 data/templates/grafana.ini.pktpl.hcl create mode 100644 data/templates/hosts.pktpl.hcl create mode 100644 pkg/templater/files.go create mode 100644 pkg/templater/persist.go create mode 100644 pkg/templater/services.go create mode 100644 pkg/templater/system_services.go create mode 100644 pkg/templater/system_users.go diff --git a/Makefile b/Makefile index 254cad2..631899d 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux cmd/templater.go + GOOS=linux go build -o bin/templater-linux cmd/templater/main.go + GOOS=linux go build -o bin/bootstraper-linux cmd/bootstraper/main.go upx bin/templater-linux upx bin/templaster-server install: cp bin/templater-linux $(DESTDIR)/bin/templater + cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper uninstall: rm $(DESTDIR)/bin/templater diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go new file mode 100644 index 0000000..150f28f --- /dev/null +++ b/cmd/bootstraper/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" +) + +func main() { + + var args struct { + Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"` + TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"` + } + + arg.MustParse(&args) + + var hostConfig templater.TemplaterConfig + + err := hostConfig.New(args.Config, args.TemplateDirectory) + if err != nil { + panic(err) + } + if err = hostConfig.ManageServices(); err != nil { + panic(err) + } +} diff --git a/cmd/templater.go b/cmd/templater/main.go similarity index 86% rename from cmd/templater.go rename to cmd/templater/main.go index 904353e..5fba371 100644 --- a/cmd/templater.go +++ b/cmd/templater/main.go @@ -55,13 +55,13 @@ func main() { config = []byte(args.Config) } - result := "" - if templateType == "go" { - result = templater.ProcessGoTemplate(templateFile, config) - } else if templateType == "hcl" { - result = templater.ProcessHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) + var file templater.ConfigFile + file.Source = templateFile + file.TemplateType = templateType + + result, err := file.ProcessTemplate(templateFile, config) + if err != nil { + panic(err) } if output == "stdout" { fmt.Printf("%s", result) diff --git a/data/config/test-services.json b/data/config/test-services.json new file mode 100644 index 0000000..60ae047 --- /dev/null +++ b/data/config/test-services.json @@ -0,0 +1,92 @@ +{ + "Name": "LokiStack", + "Global": { + "Vars": { + "EnableLoki": true, + "EnableGrafana": false, + "EnablePrometheus" : false + }, + "ConfigFiles": [ + { + "destination": "/etc/hosts", + "source": "hosts.pktpl.hcl", + "mode": "600", + "owner": "root", + "group": "root" + } + ] + }, + "Services": { + "Loki": { + "EnabledBy": { "var": "EnableLoki", "value": true }, + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mode": "600", + "owner": "loki", + "group": "grafana" + } + ], + "Vars": { + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } + }, + "Daemon": { + "name": "loki", + "enabled": true + }, + "Users": { + "loki": { + "username": "loki", + "group": "grafana", + "home" : "/srv/loki", + "shell": "/bin/nologin" + } + } + }, + "Grafana": { + "EnabledBy": { "var": "EnableGrafana", "value": true }, + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "source": "grafana.ini.pktpl.hcl", + "mode": "700", + "owner": "grafana", + "group": "grafana" + } + ], + "Vars": { + "AuthEnabled": false, + "User": "toto", + "Group": "grafana" + }, + "Users": { + "grafana": { + "username": "grafana", + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin" + } + }, + "Daemon": { + "name": "grafana", + "enabled": true + } + } + } +} \ No newline at end of file diff --git a/data/config/test-servies-2.json b/data/config/test-servies-2.json new file mode 100644 index 0000000..e20d70a --- /dev/null +++ b/data/config/test-servies-2.json @@ -0,0 +1,61 @@ +{ + "Name": "OpenNebula", + "Services": { + "Global": { + "Vars": { + "EnableOpenNebula": true + }, + "ConfigFiles": [ + { + "destination": "/etc/hosts", + "source": "hosts.pktpl.hcl", + "mode": "600", + "owner": "root", + "group": "root" + } + ] + }, + "OpenNebula": { + "EnabledBy": { "var": "EnableOpenNebula", "value": true }, + "ConfigFiles": [ + { + "destination": "/etc/one/oned.conf", + "source": "loki-local-config.pktpl.hcl", + "mode": "600", + "owner": "loki", + "group": "grafana" + } + ], + "Vars": { + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } + }, + "Daemon": { + "name": "oned", + "enabled": true + }, + "Users": { + "loki": { + "username": "oneadmin", + "group": "oneadmin", + "home" : "/var/lib/one", + "shell": "/bin/sh" + } + } + } + } +} \ No newline at end of file diff --git a/data/config/test.json b/data/config/test.json deleted file mode 100644 index 14035cb..0000000 --- a/data/config/test.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Name": "loki", - "ConfigFiles": [ - { - "destination": "/etc/loki/loki-local-config.yaml", - "source": "loki-local-config.pktpl.hcl", - "mod": "600" - } - ], - "AuthEnabled": false, - "User": "loki", - "Group": "grafana", - "HTTPPort": "3100", - "GRPCPort": "9096", - "AlertManagerURL": "http://localhost:9093", - "StorageRoot": "/var/loki", - "SharedStore": "filesystem", - "ObjectStore": "filesystem", - "LogLevel": "error", - "S3": { - "URL": "", - "BucketName": "", - "APIKey": "", - "APISecretKey": "" - } -} \ No newline at end of file diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl new file mode 100644 index 0000000..bd5f3a4 --- /dev/null +++ b/data/templates/grafana.ini.pktpl.hcl @@ -0,0 +1,5 @@ +%{ if AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } diff --git a/data/templates/hosts.pktpl.hcl b/data/templates/hosts.pktpl.hcl new file mode 100644 index 0000000..e69de29 diff --git a/pkg/templater/files.go b/pkg/templater/files.go new file mode 100644 index 0000000..5f21384 --- /dev/null +++ b/pkg/templater/files.go @@ -0,0 +1,145 @@ +package templater + +import ( + "path/filepath" + "strconv" + + "bytes" + encjson "encoding/json" + "fmt" + "os" + "text/template" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type ConfigFile struct { + Destination string `form:"destination" json:"destination"` + Source string `form:"source" json:"source"` + TemplateType string `json:"type"` + Mode string `form:"mod" json:"mode"` + Owner string `json:"owner"` + Group string `json:"group"` +} + +func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error { + var template string + dest := filepath.Join(root, cf.Destination) + source := filepath.Join(templateDir, cf.Source) + intMod, err := strconv.ParseInt(cf.Mode, 8, 64) + if err != nil { + return (err) + } + + template, err = cf.ProcessTemplate(source, values) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + dirname := filepath.Dir(dest) + err = os.MkdirAll(dirname, os.FileMode(int(0700))) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + err = os.WriteFile(dest, []byte(template), os.FileMode(intMod)) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + return nil +} + +func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) { + var result string + var err error + + fmt.Printf("Processing %s\n", source) + if cf.TemplateType == "hcl" { + result, err = cf.processHCLTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process HCL template failed with error: %v", err) + } + } else if cf.TemplateType == "go" { + result, err = cf.processGoTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process GO template failed with error: %v", err) + } + } + return result, nil +} + +func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) { + + // The JSON configuration + var confData map[string]interface{} + var res bytes.Buffer + + err := encjson.Unmarshal(configValues, &confData) + utils.CheckErr(err) + + // Read the template + data, err := os.ReadFile(file) + utils.CheckErr(err) + + tpl, err := template.New("conf").Parse(string(data)) + utils.CheckErr(err) + + utils.CheckErr(tpl.Execute(&res, confData["Config"])) + + return res.String(), nil +} + +func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) { + + fct, err := os.ReadFile(file) + utils.CheckErr(err) + + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) + + // Retrieve values from JSON + var varsVal cty.Value + ctyType, err := ctyjson.ImpliedType(config) + if err != nil { + return "", err + /* Maybe one day + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) + if diags.HasErrors() { + panic(diags.Error()) + } + varsVal, diags = cexpr.Value(&hcl.EvalContext{}) + fmt.Println(cexpr.Variables()) + checkDiags(diags) + */ + } else { + varsVal, err = ctyjson.Unmarshal(config, ctyType) + utils.CheckErr(err) + } + + ctx := &hcl.EvalContext{ + Variables: varsVal.AsValueMap(), + } + + for n := range ctx.Variables { + if !hclsyntax.ValidIdentifier(n) { + panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)) + } + } + + for _, traversal := range expr.Variables() { + root := traversal.RootName() + if _, ok := ctx.Variables[root]; !ok { + panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())) + } + } + + val, diags := expr.Value(ctx) + if diags.HasErrors() { + panic(diags.Error()) + } + + return val.AsString(), nil +} diff --git a/pkg/templater/main.go b/pkg/templater/main.go index f1fba18..8e5adb6 100644 --- a/pkg/templater/main.go +++ b/pkg/templater/main.go @@ -1,89 +1,90 @@ package templater import ( - "bytes" - encjson "encoding/json" "fmt" + "io/ioutil" "os" - "text/template" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - ctyjson "github.com/zclconf/go-cty/cty/json" - - "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" + "github.com/imdario/mergo" ) -func ProcessGoTemplate(file string, config []byte) string { +var CacheFilePath = "/var/cache/templater.db" - // The JSON configuration - var confData map[string]interface{} - var res bytes.Buffer - - err := encjson.Unmarshal(config, &confData) - utils.CheckErr(err) - - // Read the template - data, err := os.ReadFile(file) - utils.CheckErr(err) - - tpl, err := template.New("conf").Parse(string(data)) - utils.CheckErr(err) - - utils.CheckErr(tpl.Execute(&res, confData)) - - return res.String() +type SimpleCondition struct { + Var string `json:"var"` + Value bool `json:"value"` } -func ProcessHCLTemplate(file string, config []byte) string { +type TemplaterConfig struct { + Name string `json:"Name"` + TemplateDirectory string `json:"TemplateDirectory"` + Services map[string]Service `json:"Services"` + GlobalService Service `json:"Global"` +} - fct, err := os.ReadFile(file) - utils.CheckErr(err) - - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) - utils.CheckDiags(diags) - - // Retrieve values from JSON - var varsVal cty.Value - ctyType, err := ctyjson.ImpliedType(config) +func (tc *TemplaterConfig) loadCache() error { + // Load globals from cache + var cache Service + err := Load(CacheFilePath, &cache) if err != nil { - panic(err) - /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) - if diags.HasErrors() { - panic(diags.Error()) - } - varsVal, diags = cexpr.Value(&hcl.EvalContext{}) - fmt.Println(cexpr.Variables()) - checkDiags(diags) - */ - } else { - varsVal, err = ctyjson.Unmarshal(config, ctyType) - utils.CheckErr(err) + fmt.Printf("Warning: No globals to load\n") } - ctx := &hcl.EvalContext{ - Variables: varsVal.AsValueMap(), + err = mergo.Merge(&tc.GlobalService, cache) + if err != nil { + return err } - - for n := range ctx.Variables { - if !hclsyntax.ValidIdentifier(n) { - panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)) - } - } - - for _, traversal := range expr.Variables() { - root := traversal.RootName() - if _, ok := ctx.Variables[root]; !ok { - panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())) - } - } - - val, diags := expr.Value(ctx) - if diags.HasErrors() { - panic(diags.Error()) - } - - return val.AsString() + return nil +} + +func (tc *TemplaterConfig) New(confpath string, templateDir string) error { + // Load stored globals if needed + lerr := tc.loadCache() + if lerr != nil { + return lerr + } + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(confpath) + if err != nil { + return err + } + + if fileInfo.IsDir() { + // The conf path is a directory we load all the files and merge data + files, err := ioutil.ReadDir(confpath) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + for _, file := range files { + fname := fmt.Sprintf("%s/%s", confpath, file.Name()) + var ntc TemplaterConfig + err := Load(fname, &ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + err = mergo.Merge(tc, ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + } + } else { + // The conf path is a file we only load this file (of course) + err = Load(confpath, tc) + if err != nil { + return fmt.Errorf("Confiuration read failed with error: %v", err) + } + } + + tc.TemplateDirectory = templateDir + + return nil +} + +func (tc *TemplaterConfig) ManageServices() error { + for _, svr := range tc.Services { + if err := svr.Manage(tc.TemplateDirectory); err != nil { + return err + } + } + return nil } diff --git a/pkg/templater/persist.go b/pkg/templater/persist.go new file mode 100644 index 0000000..edc2155 --- /dev/null +++ b/pkg/templater/persist.go @@ -0,0 +1,51 @@ +package templater + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "sync" +) + +var lock sync.Mutex + +func marshal(v interface{}) (io.Reader, error) { + b, err := json.MarshalIndent(v, "", "\t") + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +func unmarshal(r io.Reader, v interface{}) error { + return json.NewDecoder(r).Decode(v) +} + +func Save(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("Saving Templater configuration failed with error : %v", err) + } + defer f.Close() + r, err := marshal(v) + if err != nil { + return err + } + _, err = io.Copy(f, r) + return err +} + +func Load(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return unmarshal(f, v) +} diff --git a/pkg/templater/services.go b/pkg/templater/services.go new file mode 100644 index 0000000..d31a89b --- /dev/null +++ b/pkg/templater/services.go @@ -0,0 +1,51 @@ +package templater + +import ( + "encoding/json" + "fmt" + "path/filepath" +) + +type Service struct { + EnabledBy SimpleCondition `json:"EnabledBy"` + ConfigFiles []ConfigFile `json:"ConfigFiles"` + Vars map[string]interface{} `json:"Vars"` + Daemon SystemService `json:"Daemon"` + Users map[string]SystemUser `json:"Users"` +} + +func (s *Service) Manage(templateDir string) error { + err := processConfigFiles(s.ConfigFiles, s.Vars, templateDir) + if err != nil { + return fmt.Errorf("ProcessingTemplatesFailed with error: %v", err) + } + + err = s.Daemon.Manage() + if err != nil { + return fmt.Errorf("Error managing service daemons: %v", err) + } + return nil +} + +func processConfigFiles(tpls []ConfigFile, variables map[string]interface{}, templateDir string) error { + values, err := json.Marshal(variables) + if err != nil { + return fmt.Errorf("Error unmarshaling values on template process; %v", err) + } + + for _, tpl := range tpls { + fileExt := filepath.Ext(tpl.Source) + if fileExt == ".hcl" { + tpl.TemplateType = "hcl" + } else if fileExt == ".tpl" { + tpl.TemplateType = "go" + } else { + return fmt.Errorf("Unsupported file extention %s, templates extensions have to be '.hcl' or '.tpl'", fileExt) + } + if err := tpl.Generate("/tmp/test", templateDir, values); err != nil { + return fmt.Errorf("Template %s generation failed with error %v", tpl.Source, err) + } + + } + return nil +} diff --git a/pkg/templater/system_services.go b/pkg/templater/system_services.go new file mode 100644 index 0000000..7220c74 --- /dev/null +++ b/pkg/templater/system_services.go @@ -0,0 +1,74 @@ +package templater + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemService struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Type string `json:"type"` +} + +func (sys *SystemService) SetType() { + systemdRunDirectory := "/run/systemd/system" + openRcBinaryFile := "/sbin/openrc" + + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(systemdRunDirectory) + if err == nil { + if fileInfo.IsDir() { + sys.Type = "systemd" + } + } + + fileInfo, err = os.Stat(openRcBinaryFile) + if err == nil { + if fileInfo.IsDir() { + return + } + sys.Type = "openrc" + } +} + +func (sys *SystemService) Manage() error { + if sys.Type == "" { + sys.SetType() + } + if sys.Enabled { + err := sys.Enable() + if err != nil { + return err + } + } else { + fmt.Printf("Nothing to do for daemon %s\n", sys.Name) + } + return nil +} + +func (sys *SystemService) Start() error { + fmt.Printf("Starting %s\n", sys.Name) + return nil +} + +func (sys *SystemService) Stop() error { + fmt.Printf("Stoping %s\n", sys.Name) + return nil +} + +func (sys *SystemService) Enable() error { + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "enable", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + } else if sys.Type == "openrc" { + return nil + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} diff --git a/pkg/templater/system_users.go b/pkg/templater/system_users.go new file mode 100644 index 0000000..9bcbd37 --- /dev/null +++ b/pkg/templater/system_users.go @@ -0,0 +1,8 @@ +package templater + +type SystemUser struct { + UserName string `json:"username"` + Group string `json:"group"` + Home string `json:"home"` + Shell string `json:"shell"` +} diff --git a/pkg/utils/main.go b/pkg/utils/main.go index 8ea276d..2c92b51 100644 --- a/pkg/utils/main.go +++ b/pkg/utils/main.go @@ -1,6 +1,9 @@ package utils import ( + "bytes" + "os/exec" + "github.com/hashicorp/hcl/v2" ) @@ -15,3 +18,15 @@ func CheckDiags(diag hcl.Diagnostics) { panic(diag.Error()) } } + +// Execute a system command ... +func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) { + var stdOut bytes.Buffer + var stdErr bytes.Buffer + + cmd := exec.Command(name, arg...) + cmd.Stderr = &stdErr + cmd.Stdout = &stdOut + err := cmd.Run() + return stdOut.Bytes(), stdErr.Bytes(), err +}