pcaseiro
/
templatefile
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 4 Wiki Activity

Complete rewrite of the bootstraper tool

This commit is contained in:
Philippe Caseiro 2022-06-15 17:29:45 +02:00
parent 5085bd4d69
commit 8400edc0e3
17 changed files with 617 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -2,12 +2,14 @@ LINT_ARGS ?= ./...
DESTDIR ?= "/usr/local"
bin:
GOOS=linux go build -o bin/templater-linux cmd/templater.go
GOOS=linux go build -o bin/templater-linux cmd/templater/main.go
GOOS=linux go build -o bin/bootstraper-linux cmd/bootstraper/main.go
upx bin/templater-linux
upx bin/templaster-server
install:
cp bin/templater-linux $(DESTDIR)/bin/templater
cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper
uninstall:
rm $(DESTDIR)/bin/templater

26
cmd/bootstraper/main.go Normal file
View File

@ -0,0 +1,26 @@
package main
import (
"forge.cadoles.com/pcaseiro/templatefile/pkg/templater"
"github.com/alexflint/go-arg"
)
func main() {
var args struct {
Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"`
TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"`
}
arg.MustParse(&args)
var hostConfig templater.TemplaterConfig
err := hostConfig.New(args.Config, args.TemplateDirectory)
if err != nil {
panic(err)
}
if err = hostConfig.ManageServices(); err != nil {
panic(err)
}
}

14
cmd/templater.go → cmd/templater/main.go
View File

@ -55,13 +55,13 @@ func main() {
config = []byte(args.Config)
}
result := ""
if templateType == "go" {
result = templater.ProcessGoTemplate(templateFile, config)
} else if templateType == "hcl" {
result = templater.ProcessHCLTemplate(templateFile, config)
} else {
panic(fmt.Errorf("Unsupported template type"))
var file templater.ConfigFile
file.Source = templateFile
file.TemplateType = templateType
result, err := file.ProcessTemplate(templateFile, config)
if err != nil {
panic(err)
}
if output == "stdout" {
fmt.Printf("%s", result)

92
data/config/test-services.json Normal file
View File

@ -0,0 +1,92 @@
{
"Name": "LokiStack",
"Global": {
"Vars": {
"EnableLoki": true,
"EnableGrafana": false,
"EnablePrometheus" : false
},
"ConfigFiles": [
{
"destination": "/etc/hosts",
"source": "hosts.pktpl.hcl",
"mode": "600",
"owner": "root",
"group": "root"
}
]
},
"Services": {
"Loki": {
"EnabledBy": { "var": "EnableLoki", "value": true },
"ConfigFiles": [
{
"destination": "/etc/loki/loki-local-config.yaml",
"source": "loki-local-config.pktpl.hcl",
"mode": "600",
"owner": "loki",
"group": "grafana"
}
],
"Vars": {
"AuthEnabled": false,
"User": "loki",
"Group": "grafana",
"HTTPPort": "3100",
"GRPCPort": "9096",
"AlertManagerURL": "http://localhost:9093",
"StorageRoot": "/var/loki",
"SharedStore": "filesystem",
"ObjectStore": "filesystem",
"LogLevel": "error",
"S3": {
"URL": "",
"BucketName": "",
"APIKey": "",
"APISecretKey": ""
}
},
"Daemon": {
"name": "loki",
"enabled": true
},
"Users": {
"loki": {
"username": "loki",
"group": "grafana",
"home" : "/srv/loki",
"shell": "/bin/nologin"
}
}
},
"Grafana": {
"EnabledBy": { "var": "EnableGrafana", "value": true },
"ConfigFiles": [
{
"destination": "/etc/grafana.ini",
"source": "grafana.ini.pktpl.hcl",
"mode": "700",
"owner": "grafana",
"group": "grafana"
}
],
"Vars": {
"AuthEnabled": false,
"User": "toto",
"Group": "grafana"
},
"Users": {
"grafana": {
"username": "grafana",
"group": "grafana",
"home": "/srv/grafana",
"shell": "/bin/nologin"
}
},
"Daemon": {
"name": "grafana",
"enabled": true
}
}
}
}

61
data/config/test-servies-2.json Normal file
View File

@ -0,0 +1,61 @@
{
"Name": "OpenNebula",
"Services": {
"Global": {
"Vars": {
"EnableOpenNebula": true
},
"ConfigFiles": [
{
"destination": "/etc/hosts",
"source": "hosts.pktpl.hcl",
"mode": "600",
"owner": "root",
"group": "root"
}
]
},
"OpenNebula": {
"EnabledBy": { "var": "EnableOpenNebula", "value": true },
"ConfigFiles": [
{
"destination": "/etc/one/oned.conf",
"source": "loki-local-config.pktpl.hcl",
"mode": "600",
"owner": "loki",
"group": "grafana"
}
],
"Vars": {
"AuthEnabled": false,
"User": "loki",
"Group": "grafana",
"HTTPPort": "3100",
"GRPCPort": "9096",
"AlertManagerURL": "http://localhost:9093",
"StorageRoot": "/var/loki",
"SharedStore": "filesystem",
"ObjectStore": "filesystem",
"LogLevel": "error",
"S3": {
"URL": "",
"BucketName": "",
"APIKey": "",
"APISecretKey": ""
}
},
"Daemon": {
"name": "oned",
"enabled": true
},
"Users": {
"loki": {
"username": "oneadmin",
"group": "oneadmin",
"home" : "/var/lib/one",
"shell": "/bin/sh"
}
}
}
}
}

26
data/config/test.json
View File

@ -1,26 +0,0 @@
{
"Name": "loki",
"ConfigFiles": [
{
"destination": "/etc/loki/loki-local-config.yaml",
"source": "loki-local-config.pktpl.hcl",
"mod": "600"
}
],
"AuthEnabled": false,
"User": "loki",
"Group": "grafana",
"HTTPPort": "3100",
"GRPCPort": "9096",
"AlertManagerURL": "http://localhost:9093",
"StorageRoot": "/var/loki",
"SharedStore": "filesystem",
"ObjectStore": "filesystem",
"LogLevel": "error",
"S3": {
"URL": "",
"BucketName": "",
"APIKey": "",
"APISecretKey": ""
}
}

0
data/schema/templater.hcl Normal file
View File

0
data/schema/templater.json Normal file
View File

5
data/templates/grafana.ini.pktpl.hcl Normal file
View File

@ -0,0 +1,5 @@
%{ if AuthEnabled ~}
auth_enabled: true
%{ else }
auth_enabled: false
%{ endif }

0
data/templates/hosts.pktpl.hcl Normal file
View File

151
pkg/templater/files.go Normal file
View File

@ -0,0 +1,151 @@
package templater
import (
"path/filepath"
"strconv"
"bytes"
encjson "encoding/json"
"fmt"
"os"
"text/template"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
ctyjson "github.com/zclconf/go-cty/cty/json"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
)
type ConfigFile struct {
Destination string `form:"destination" json:"destination"` // Where do we write the configuration file
Source string `form:"source" json:"source"` // The template file short name
TemplateType string `json:"type"` // The template file type (hcl or gotemplate)
Mode string `form:"mod" json:"mode"` // The configuration file final permissions (mode)
Owner string `json:"owner"` // The configuration file owner
Group string `json:"group"` // The configuration file group owner
}
// Generate the configuration file from the template (hcl or json)
func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error {
var template string
dest := filepath.Join(root, cf.Destination)
source := filepath.Join(templateDir, cf.Source)
intMod, err := strconv.ParseInt(cf.Mode, 8, 64)
if err != nil {
return (err)
}
template, err = cf.ProcessTemplate(source, values)
if err != nil {
return fmt.Errorf("Process templates failed with error: %v", err)
}
dirname := filepath.Dir(dest)
err = os.MkdirAll(dirname, os.FileMode(int(0700)))
if err != nil {
return fmt.Errorf("Process templates failed with error: %v", err)
}
err = os.WriteFile(dest, []byte(template), os.FileMode(intMod))
if err != nil {
return fmt.Errorf("Process templates failed with error: %v", err)
}
return nil
}
// Process the template with the provided values
func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) {
var result string
var err error
fmt.Printf("Processing %s\n", source)
if cf.TemplateType == "hcl" {
// The template is an hcl template so we call processHCLTemplate
result, err = cf.processHCLTemplate(source, values)
if err != nil {
return "", fmt.Errorf("Process HCL template failed with error: %v", err)
}
} else if cf.TemplateType == "go" {
// The template is a go template so we call processGoTemplate
result, err = cf.processGoTemplate(source, values)
if err != nil {
return "", fmt.Errorf("Process GO template failed with error: %v", err)
}
}
return result, nil
}
// The actual template processing for Go templates
func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) {
// The JSON configuration
var confData map[string]interface{}
var res bytes.Buffer
err := encjson.Unmarshal(configValues, &confData)
utils.CheckErr(err)
// Read the template
data, err := os.ReadFile(file)
utils.CheckErr(err)
tpl, err := template.New("conf").Parse(string(data))
utils.CheckErr(err)
utils.CheckErr(tpl.Execute(&res, confData["Config"]))
return res.String(), nil
}
// The actual template processing for HCL templates
func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) {
fct, err := os.ReadFile(file)
utils.CheckErr(err)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
utils.CheckDiags(diags)
// Retrieve values from JSON
var varsVal cty.Value
ctyType, err := ctyjson.ImpliedType(config)
if err != nil {
return "", err
/* Maybe one day
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
if diags.HasErrors() {
panic(diags.Error())
}
varsVal, diags = cexpr.Value(&hcl.EvalContext{})
fmt.Println(cexpr.Variables())
checkDiags(diags)
*/
} else {
varsVal, err = ctyjson.Unmarshal(config, ctyType)
utils.CheckErr(err)
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n))
}
}
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()))
}
}
val, diags := expr.Value(ctx)
if diags.HasErrors() {
panic(diags.Error())
}
return val.AsString(), nil
}

145
pkg/templater/main.go
View File

@ -1,89 +1,90 @@
package templater
import (
"bytes"
encjson "encoding/json"
"fmt"
"io/ioutil"
"os"
"text/template"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
ctyjson "github.com/zclconf/go-cty/cty/json"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
"github.com/imdario/mergo"
)
func ProcessGoTemplate(file string, config []byte) string {
var CacheFilePath = "/var/cache/templater.db"
// The JSON configuration
var confData map[string]interface{}
var res bytes.Buffer
err := encjson.Unmarshal(config, &confData)
utils.CheckErr(err)
// Read the template
data, err := os.ReadFile(file)
utils.CheckErr(err)
tpl, err := template.New("conf").Parse(string(data))
utils.CheckErr(err)
utils.CheckErr(tpl.Execute(&res, confData))
return res.String()
type SimpleCondition struct {
Var string `json:"var"`
Value bool `json:"value"`
}
func ProcessHCLTemplate(file string, config []byte) string {
type TemplaterConfig struct {
Name string `json:"Name"`
TemplateDirectory string `json:"TemplateDirectory"`
Services map[string]Service `json:"Services"`
GlobalService Service `json:"Global"`
}
fct, err := os.ReadFile(file)
utils.CheckErr(err)
expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1})
utils.CheckDiags(diags)
// Retrieve values from JSON
var varsVal cty.Value
ctyType, err := ctyjson.ImpliedType(config)
func (tc *TemplaterConfig) loadCache() error {
// Load globals from cache
var cache Service
err := Load(CacheFilePath, &cache)
if err != nil {
panic(err)
/* Maybe one day
cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1})
if diags.HasErrors() {
panic(diags.Error())
}
varsVal, diags = cexpr.Value(&hcl.EvalContext{})
fmt.Println(cexpr.Variables())
checkDiags(diags)
*/
} else {
varsVal, err = ctyjson.Unmarshal(config, ctyType)
utils.CheckErr(err)
fmt.Printf("Warning: No globals to load\n")
}
ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
err = mergo.Merge(&tc.GlobalService, cache)
if err != nil {
return err
}
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n))
}
}
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()))
}
}
val, diags := expr.Value(ctx)
if diags.HasErrors() {
panic(diags.Error())
}
return val.AsString()
return nil
}
func (tc *TemplaterConfig) New(confpath string, templateDir string) error {
// Load stored globals if needed
lerr := tc.loadCache()
if lerr != nil {
return lerr
}
// Check if the configuration path is a Directory or a file
fileInfo, err := os.Stat(confpath)
if err != nil {
return err
}
if fileInfo.IsDir() {
// The conf path is a directory we load all the files and merge data
files, err := ioutil.ReadDir(confpath)
if err != nil {
return fmt.Errorf("Templater configuration load failed with error: %v", err)
}
for _, file := range files {
fname := fmt.Sprintf("%s/%s", confpath, file.Name())
var ntc TemplaterConfig
err := Load(fname, &ntc)
if err != nil {
return fmt.Errorf("Templater configuration load failed with error: %v", err)
}
err = mergo.Merge(tc, ntc)
if err != nil {
return fmt.Errorf("Templater configuration load failed with error: %v", err)
}
}
} else {
// The conf path is a file we only load this file (of course)
err = Load(confpath, tc)
if err != nil {
return fmt.Errorf("Confiuration read failed with error: %v", err)
}
}
tc.TemplateDirectory = templateDir
return nil
}
func (tc *TemplaterConfig) ManageServices() error {
for _, svr := range tc.Services {
if err := svr.Manage(tc.TemplateDirectory); err != nil {
return err
}
}
return nil
}

51
pkg/templater/persist.go Normal file
View File

@ -0,0 +1,51 @@
package templater
import (
"bytes"
"encoding/json"
"fmt"
"io"
"os"
"sync"
)
var lock sync.Mutex
func marshal(v interface{}) (io.Reader, error) {
b, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return nil, err
}
return bytes.NewReader(b), nil
}
func unmarshal(r io.Reader, v interface{}) error {
return json.NewDecoder(r).Decode(v)
}
func Save(path string, v interface{}) error {
lock.Lock()
defer lock.Unlock()
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("Saving Templater configuration failed with error : %v", err)
}
defer f.Close()
r, err := marshal(v)
if err != nil {
return err
}
_, err = io.Copy(f, r)
return err
}
func Load(path string, v interface{}) error {
lock.Lock()
defer lock.Unlock()
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return unmarshal(f, v)
}

51
pkg/templater/services.go Normal file
View File

@ -0,0 +1,51 @@
package templater
import (
"encoding/json"
"fmt"
"path/filepath"
)
type Service struct {
EnabledBy SimpleCondition `json:"EnabledBy"`
ConfigFiles []ConfigFile `json:"ConfigFiles"`
Vars map[string]interface{} `json:"Vars"`
Daemon SystemService `json:"Daemon"`
Users map[string]SystemUser `json:"Users"`
}
func (s *Service) Manage(templateDir string) error {
err := processConfigFiles(s.ConfigFiles, s.Vars, templateDir)
if err != nil {
return fmt.Errorf("ProcessingTemplatesFailed with error: %v", err)
}
err = s.Daemon.Manage()
if err != nil {
return fmt.Errorf("Error managing service daemons: %v", err)
}
return nil
}
func processConfigFiles(tpls []ConfigFile, variables map[string]interface{}, templateDir string) error {
values, err := json.Marshal(variables)
if err != nil {
return fmt.Errorf("Error unmarshaling values on template process; %v", err)
}
for _, tpl := range tpls {
fileExt := filepath.Ext(tpl.Source)
if fileExt == ".hcl" {
tpl.TemplateType = "hcl"
} else if fileExt == ".tpl" {
tpl.TemplateType = "go"
} else {
return fmt.Errorf("Unsupported file extention %s, templates extensions have to be '.hcl' or '.tpl'", fileExt)
}
if err := tpl.Generate("/tmp/test", templateDir, values); err != nil {
return fmt.Errorf("Template %s generation failed with error %v", tpl.Source, err)
}
}
return nil
}

74
pkg/templater/system_services.go Normal file
View File

@ -0,0 +1,74 @@
package templater
import (
"fmt"
"os"
"forge.cadoles.com/pcaseiro/templatefile/pkg/utils"
)
type SystemService struct {
Name string `json:"name"`
Enabled bool `json:"enabled"`
Type string `json:"type"`
}
func (sys *SystemService) SetType() {
systemdRunDirectory := "/run/systemd/system"
openRcBinaryFile := "/sbin/openrc"
// Check if the configuration path is a Directory or a file
fileInfo, err := os.Stat(systemdRunDirectory)
if err == nil {
if fileInfo.IsDir() {
sys.Type = "systemd"
}
}
fileInfo, err = os.Stat(openRcBinaryFile)
if err == nil {
if fileInfo.IsDir() {
return
}
sys.Type = "openrc"
}
}
func (sys *SystemService) Manage() error {
if sys.Type == "" {
sys.SetType()
}
if sys.Enabled {
err := sys.Enable()
if err != nil {
return err
}
} else {
fmt.Printf("Nothing to do for daemon %s\n", sys.Name)
}
return nil
}
func (sys *SystemService) Start() error {
fmt.Printf("Starting %s\n", sys.Name)
return nil
}
func (sys *SystemService) Stop() error {
fmt.Printf("Stoping %s\n", sys.Name)
return nil
}
func (sys *SystemService) Enable() error {
if sys.Type == "systemd" {
_, stdErr, err := utils.RunSystemCommand("systemctl", "enable", sys.Name)
if err != nil {
return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr)
}
} else if sys.Type == "openrc" {
return nil
} else {
return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name)
}
return nil
}

8
pkg/templater/system_users.go Normal file
View File

@ -0,0 +1,8 @@
package templater
type SystemUser struct {
UserName string `json:"username"`
Group string `json:"group"`
Home string `json:"home"`
Shell string `json:"shell"`
}

15
pkg/utils/main.go
View File

@ -1,6 +1,9 @@
package utils
import (
"bytes"
"os/exec"
"github.com/hashicorp/hcl/v2"
)
@ -15,3 +18,15 @@ func CheckDiags(diag hcl.Diagnostics) {
panic(diag.Error())
}
}
// Execute a system command ...
func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) {
var stdOut bytes.Buffer
var stdErr bytes.Buffer
cmd := exec.Command(name, arg...)
cmd.Stderr = &stdErr
cmd.Stdout = &stdOut
err := cmd.Run()
return stdOut.Bytes(), stdErr.Bytes(), err
}