diff --git a/.gitignore b/.gitignore index f4d432a..5bbb8bf 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ # Dependency directories (remove the comment below to include it) # vendor/ - +bin/ diff --git a/Makefile b/Makefile index 254cad2..a8b6c47 100644 --- a/Makefile +++ b/Makefile @@ -2,12 +2,14 @@ LINT_ARGS ?= ./... DESTDIR ?= "/usr/local" bin: - GOOS=linux go build -o bin/templater-linux cmd/templater.go + GOOS=linux CGO_ENABLED=0 go build -o bin/templater-linux cmd/templater/main.go + GOOS=linux CGO_ENABLED=0 go build -o bin/bootstraper-linux cmd/bootstraper/main.go upx bin/templater-linux - upx bin/templaster-server + upx bin/bootstraper-linux install: cp bin/templater-linux $(DESTDIR)/bin/templater + cp bin/bootstraper-linux $(DESTDIR)/bin/bootstraper uninstall: rm $(DESTDIR)/bin/templater diff --git a/api/main.go b/api/main.go index 7c12c62..d0766f1 100644 --- a/api/main.go +++ b/api/main.go @@ -1,9 +1,6 @@ package api import ( - "net/http" - - "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" "github.com/gin-gonic/gin" ) @@ -13,6 +10,12 @@ type Template struct { Config string } +func Generate(c *gin.Context) { + return +} + +/* + func Generate(c *gin.Context) { var template Template @@ -42,3 +45,4 @@ func Generate(c *gin.Context) { } } +*/ diff --git a/cmd/bootstraper/main.go b/cmd/bootstraper/main.go new file mode 100644 index 0000000..68f0732 --- /dev/null +++ b/cmd/bootstraper/main.go @@ -0,0 +1,27 @@ +package main + +import ( + "forge.cadoles.com/pcaseiro/templatefile/pkg/templater" + "github.com/alexflint/go-arg" +) + +func main() { + + var args struct { + Config string `arg:"-c,--config,env:CONFIG" help:"Configuration values file or directory path" default:"./data/config"` + TemplateDirectory string `arg:"-t,--template-dir,env:TEMPLATE_DIR" help:"Template directory path" default:"./data/templates"` + RootDirectory string `arg:"-r,--root-dir,env:ROOT_DIR" help:"Generate files with this root instead of /" default:"/"` + } + + arg.MustParse(&args) + + var hostConfig templater.TemplaterConfig + + err := hostConfig.New(args.Config, args.TemplateDirectory) + if err != nil { + panic(err) + } + if err = hostConfig.ManageServices(args.RootDirectory); err != nil { + panic(err) + } +} diff --git a/cmd/templater.go b/cmd/templater/main.go similarity index 86% rename from cmd/templater.go rename to cmd/templater/main.go index 904353e..5fba371 100644 --- a/cmd/templater.go +++ b/cmd/templater/main.go @@ -55,13 +55,13 @@ func main() { config = []byte(args.Config) } - result := "" - if templateType == "go" { - result = templater.ProcessGoTemplate(templateFile, config) - } else if templateType == "hcl" { - result = templater.ProcessHCLTemplate(templateFile, config) - } else { - panic(fmt.Errorf("Unsupported template type")) + var file templater.ConfigFile + file.Source = templateFile + file.TemplateType = templateType + + result, err := file.ProcessTemplate(templateFile, config) + if err != nil { + panic(err) } if output == "stdout" { fmt.Printf("%s", result) diff --git a/data/config/test-services.hcl b/data/config/test-services.hcl new file mode 100644 index 0000000..80d5aee --- /dev/null +++ b/data/config/test-services.hcl @@ -0,0 +1,127 @@ +LokiStack = { + Name: "LokiStack" + Global: { + Vars: {} + ConfigFiles: [ + { + destination: /etc/hosts + source: hosts.pktpl.hcl + mode: 600 + owner: root + group: root + } + ] + } + Services: { + Loki: { + ConfigFiles: [ + { + destination: /etc/loki/loki-local-config.yaml + source: loki-local-config.pktpl.hcl + mode: 600 + owner: loki + service: loki + group: grafana + } + ] + Repositories: { + Grafana: { + type: helm + name: grafana + url: https://grafana.github.io/helm-charts + enabled:true + } + AlpineTesting: { + type: apk + name: testing + url: http://mirrors.bfsu.edu.cn/alpine/edge/testing + enabled: true + } + } + Packages: { + loki: { + name: loki + action: install + } + promtail: { + name: loki-promtail + action: install + } + loki-helm: { + type: helm + name: loki + repo: grafana/loki-simple-scalable + } + } + Vars: { + AuthEnabled: false + User: loki + Group: grafana + HTTPPort: 3100 + GRPCPort: 9096 + AlertManagerURL: http://localhost:9093 + StorageRoot: /var/loki + SharedStore: filesystem + ObjectStore: filesystem + LogLevel: error + S3: { + URL: + BucketName: + APIKey: + APISecretKey: + } + } + Daemons: { + Loki: { + name: loki + enabled: true + } + } + Users: { + loki: { + username: loki + group: grafana + home : /srv/loki + shell: /bin/nologin + } + } + } + Grafana: { + ConfigFiles: [ + { + destination: /etc/grafana.ini + source: grafana.ini.pktpl.hcl + mode: 700 + owner: grafana + group: grafana + } + ] + Packages: { + grafana: { + name: grafana + action: install + } + } + Vars: { + AuthEnabled: false + User: toto + Group: grafana + } + Users: { + grafana: { + username: grafana + group: grafana + home: /srv/grafana + shell: /bin/nologin + } + } + Daemons: { + grafana: { + name: grafana + type: auto + enabled: true + } + } + } + } +} \ No newline at end of file diff --git a/data/config/test-services.json b/data/config/test-services.json new file mode 100644 index 0000000..66855d4 --- /dev/null +++ b/data/config/test-services.json @@ -0,0 +1,127 @@ +{ + "Name": "LokiStack", + "Global": { + "Vars": {}, + "ConfigFiles": [ + { + "destination": "/etc/hosts", + "source": "hosts.pktpl.hcl", + "mode": "600", + "owner": "root", + "group": "root" + } + ] + }, + "Services": { + "Loki": { + "ConfigFiles": [ + { + "destination": "/etc/loki/loki-local-config.yaml", + "source": "loki-local-config.pktpl.hcl", + "mode": "600", + "owner": "loki", + "service": "loki", + "group": "grafana" + } + ], + "Repositories": { + "Grafana": { + "type": "helm", + "name": "grafana", + "url": "https://grafana.github.io/helm-charts", + "enabled":true + }, + "AlpineTesting": { + "type": "apk", + "name": "testing", + "url": "http://mirrors.bfsu.edu.cn/alpine/edge/testing", + "enabled": true + } + }, + "Packages": { + "loki": { + "name": "loki", + "action": "install" + }, + "promtail": { + "name": "loki-promtail", + "action": "install" + }, + "loki-helm": { + "type": "helm", + "name": "loki", + "repo": "grafana/loki-simple-scalable" + } + }, + "Vars": { + "AuthEnabled": false, + "User": "loki", + "Group": "grafana", + "HTTPPort": "3100", + "GRPCPort": "9096", + "AlertManagerURL": "http://localhost:9093", + "StorageRoot": "/var/loki", + "SharedStore": "filesystem", + "ObjectStore": "filesystem", + "LogLevel": "error", + "S3": { + "URL": "", + "BucketName": "", + "APIKey": "", + "APISecretKey": "" + } + }, + "Daemons": { + "Loki": { + "name": "loki", + "enabled": true + } + }, + "Users": { + "loki": { + "username": "loki", + "group": "grafana", + "home" : "/srv/loki", + "shell": "/bin/nologin" + } + } + }, + "Grafana": { + "ConfigFiles": [ + { + "destination": "/etc/grafana.ini", + "source": "grafana.ini.pktpl.hcl", + "mode": "700", + "owner": "grafana", + "group": "grafana" + } + ], + "Packages": { + "grafana": { + "name": "grafana", + "action": "install" + } + }, + "Vars": { + "AuthEnabled": false, + "User": "toto", + "Group": "grafana" + }, + "Users": { + "grafana": { + "username": "grafana", + "group": "grafana", + "home": "/srv/grafana", + "shell": "/bin/nologin" + } + }, + "Daemons": { + "grafana": { + "name": "grafana", + "type": "auto", + "enabled": true + } + } + } + } +} \ No newline at end of file diff --git a/data/config/test.json b/data/config/test.json deleted file mode 100644 index 14035cb..0000000 --- a/data/config/test.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Name": "loki", - "ConfigFiles": [ - { - "destination": "/etc/loki/loki-local-config.yaml", - "source": "loki-local-config.pktpl.hcl", - "mod": "600" - } - ], - "AuthEnabled": false, - "User": "loki", - "Group": "grafana", - "HTTPPort": "3100", - "GRPCPort": "9096", - "AlertManagerURL": "http://localhost:9093", - "StorageRoot": "/var/loki", - "SharedStore": "filesystem", - "ObjectStore": "filesystem", - "LogLevel": "error", - "S3": { - "URL": "", - "BucketName": "", - "APIKey": "", - "APISecretKey": "" - } -} \ No newline at end of file diff --git a/data/schema/templater.hcl b/data/schema/templater.hcl new file mode 100644 index 0000000..e69de29 diff --git a/data/schema/templater.json b/data/schema/templater.json new file mode 100644 index 0000000..e69de29 diff --git a/data/templates/grafana.ini.pktpl.hcl b/data/templates/grafana.ini.pktpl.hcl new file mode 100644 index 0000000..3ba92f6 --- /dev/null +++ b/data/templates/grafana.ini.pktpl.hcl @@ -0,0 +1,5 @@ +%{ if Vars.AuthEnabled ~} +auth_enabled: true +%{ else } +auth_enabled: false +%{ endif } \ No newline at end of file diff --git a/data/templates/hosts.pktpl.hcl b/data/templates/hosts.pktpl.hcl new file mode 100644 index 0000000..e69de29 diff --git a/data/templates/loki-local-config.pktpl.hcl b/data/templates/loki-local-config.pktpl.hcl index 9fd10f7..0e64c43 100644 --- a/data/templates/loki-local-config.pktpl.hcl +++ b/data/templates/loki-local-config.pktpl.hcl @@ -1,18 +1,18 @@ -%{ if AuthEnabled ~} +%{ if Vars.AuthEnabled ~} auth_enabled: true %{ else } auth_enabled: false %{ endif } server: - http_listen_port: ${HTTPPort} - grpc_listen_port: ${GRPCPort} - log_level: ${LogLevel} + http_listen_port: ${Vars.HTTPPort} + grpc_listen_port: ${Vars.GRPCPort} + log_level: ${Vars.LogLevel} ingester: wal: enabled: true - dir: ${StorageRoot}/wal + dir: ${Vars.StorageRoot}/wal flush_on_shutdown: true lifecycler: address: 127.0.0.1 @@ -23,7 +23,7 @@ ingester: final_sleep: 0s chunk_idle_period: 1h # Any chunk not receiving new logs in this time will be flushed max_chunk_age: 1h # All chunks will be flushed when they hit this age, default is 1h - chunk_target_size: 1048576 # Loki will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first + chunk_target_size: 1048576 # Vars. will attempt to build chunks up to 1.5MB, flushing first if chunk_idle_period or max_chunk_age is reached first chunk_retain_period: 30s # Must be greater than index read cache TTL if using an index cache (Default index read cache TTL is 5m) max_transfer_retries: 0 # Chunk transfers disabled @@ -31,31 +31,31 @@ schema_config: configs: - from: 2020-05-15 store: boltdb-shipper - object_store: ${ObjectStore} + object_store: ${Vars.ObjectStore} schema: v11 index: prefix: index_ period: 24h storage_config: - boltdb_shipper: - active_index_directory: ${StorageRoot}/index - shared_store: ${SharedStore} - cache_location: ${StorageRoot}/cache - cache_ttl: 168h + boltdb_shipper: + active_index_directory: ${Vars.StorageRoot}/index + shared_store: ${Vars.SharedStore} + cache_location: ${Vars.StorageRoot}/cache + cache_ttl: 168h -%{ if ObjectStore == "filesystem" ~} +%{ if Vars.ObjectStore == "filesystem" ~} filesystem: - directory: ${StorageRoot}/chunks + directory: ${Vars.StorageRoot}/chunks %{ else } aws: - s3: s3://${S3.APIKey}:${S3.APISecretKey}@${S3.URL}/${S3.BucketName} + s3: s3://${Vars.S3.APIKey}:${Vars.S3.APISecretKey}@${Vars.S3.URL}/${Vars.S3.BucketName} s3forcepathstyle: true %{ endif } compactor: - shared_store: ${SharedStore} - working_directory: ${StorageRoot}/compactor + shared_store: ${Vars.SharedStore} + working_directory: ${Vars.StorageRoot}/compactor compaction_interval: 10m limits_config: @@ -73,10 +73,10 @@ ruler: storage: type: local local: - directory: ${StorageRoot}/rules - rule_path: ${StorageRoot}/rules - alertmanager_url: ${AlertManagerURL} + directory: ${Vars.StorageRoot}/rules + rule_path: ${Vars.StorageRoot}/rules + alertmanager_url: ${Vars.AlertManagerURL} ring: kvstore: store: inmemory - enable_api: true \ No newline at end of file + enable_api: true diff --git a/data/templates/loki-local-config.tpl b/data/templates/loki-local-config.tpl index 07cd8c8..a0de866 100644 --- a/data/templates/loki-local-config.tpl +++ b/data/templates/loki-local-config.tpl @@ -1,18 +1,18 @@ -{{ if .AuthEnabled }} +{{ if .Vars.AuthEnabled }} auth_enabled: true {{ else }} auth_enabled: false {{ end }} server: - http_listen_port: {{ .HTTPPort }} - grpc_listen_port: {{ .GRPCPort }} + http_listen_port: {{ .Vars.HTTPPort }} + grpc_listen_port: {{ .Vars.GRPCPort }} log_level: {{ .LogLevel }} ingester: wal: enabled: true - dir: {{ .StorageRoot }}/wal + dir: {{ .Vars.StorageRoot }}/wal flush_on_shutdown: true lifecycler: address: 127.0.0.1 @@ -31,7 +31,7 @@ schema_config: configs: - from: 2020-05-15 store: boltdb-shipper - object_store: {{ .ObjectStore }} + object_store: {{ .Vars.ObjectStore }} schema: v11 index: prefix: index_ @@ -39,23 +39,23 @@ schema_config: storage_config: boltdb_shipper: - active_index_directory: {{ .StorageRoot }}/index - shared_store: {{ .SharedStore }} - cache_location: {{ .StorageRoot }}/cache + active_index_directory: {{ .Vars.StorageRoot }}/index + shared_store: {{ .Vars.SharedStore }} + cache_location: {{ .Vars.StorageRoot }}/cache cache_ttl: 168h {{ if eq (.ObjectStore) ("filesystem") }} filesystem: - directory: {{ .StorageRoot }}/chunks + directory: {{ .Vars.StorageRoot }}/chunks {{ else }} aws: - s3: s3://{{ .S3.APIKey }}:{{ .S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} + s3: s3://{{ .Vars.S3.APIKey }}:{{ .Vars.S3.APISecretKey}}@{{ .S3.URL}}/{{ .S3.BucketName}} s3forcepathstyle: true {{ end }} compactor: - shared_store: {{ .SharedStore }} - working_directory: {{ .StorageRoot }}/compactor + shared_store: {{ .Vars.SharedStore }} + working_directory: {{ .Vars.StorageRoot }}/compactor compaction_interval: 10m limits_config: @@ -73,9 +73,9 @@ ruler: storage: type: local local: - directory: {{ .StorageRoot }}/rules - rule_path: {{ .StorageRoot }}/rules - alertmanager_url: {{ .AlertManagerURL }} + directory: {{ .Vars.StorageRoot }}/rules + rule_path: {{ .Vars.StorageRoot }}/rules + alertmanager_url: {{ .Vars.AlertManagerURL }} ring: kvstore: store: inmemory diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..71790bf --- /dev/null +++ b/go.mod @@ -0,0 +1,38 @@ +module forge.cadoles.com/pcaseiro/templatefile + +go 1.18 + +require ( + github.com/alexflint/go-arg v1.4.3 + github.com/gin-gonic/gin v1.8.1 + github.com/hashicorp/hcl/v2 v2.11.1 + github.com/imdario/mergo v0.3.13 + github.com/zclconf/go-cty v1.10.0 + gopkg.in/ini.v1 v1.66.6 +) + +require ( + github.com/agext/levenshtein v1.2.1 // indirect + github.com/alexflint/go-scalar v1.1.0 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/go-playground/validator/v10 v10.10.0 // indirect + github.com/goccy/go-json v0.9.7 // indirect + github.com/google/go-cmp v0.5.8 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/ugorji/go/codec v1.2.7 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 // indirect + golang.org/x/text v0.3.6 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect +) diff --git a/pkg/templater/files.go b/pkg/templater/files.go new file mode 100644 index 0000000..c5428ab --- /dev/null +++ b/pkg/templater/files.go @@ -0,0 +1,153 @@ +package templater + +import ( + "log" + "path/filepath" + "strconv" + + "bytes" + encjson "encoding/json" + "fmt" + "os" + "text/template" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/zclconf/go-cty/cty" + ctyjson "github.com/zclconf/go-cty/cty/json" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type ConfigFile struct { + Destination string `form:"destination" json:"destination"` // Where do we write the configuration file + Source string `form:"source" json:"source"` // The template file short name + TemplateType string `json:"type"` // The template file type (hcl or gotemplate) + Mode string `form:"mod" json:"mode"` // The configuration file final permissions (mode) + Owner string `json:"owner"` // The configuration file owner + Service string `json:"service"` // Service to restart after configuration generation + Group string `json:"group"` // The configuration file group owner +} + +// Generate the configuration file from the template (hcl or json) +func (cf *ConfigFile) Generate(root string, templateDir string, values []byte) error { + var template string + dest := filepath.Join(root, cf.Destination) + source := filepath.Join(templateDir, cf.Source) + intMod, err := strconv.ParseInt(cf.Mode, 8, 64) + if err != nil { + return (err) + } + + template, err = cf.ProcessTemplate(source, values) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + dirname := filepath.Dir(dest) + err = os.MkdirAll(dirname, os.FileMode(int(0700))) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + err = os.WriteFile(dest, []byte(template), os.FileMode(intMod)) + if err != nil { + return fmt.Errorf("Process templates failed with error: %v", err) + } + log.Printf("\tFile %s generated\n", dest) + return nil +} + +// Process the template with the provided values +func (cf *ConfigFile) ProcessTemplate(source string, values []byte) (string, error) { + var result string + var err error + + if cf.TemplateType == "hcl" { + // The template is an hcl template so we call processHCLTemplate + result, err = cf.processHCLTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process HCL template failed with error: %v", err) + } + } else if cf.TemplateType == "go" { + // The template is a go template so we call processGoTemplate + result, err = cf.processGoTemplate(source, values) + if err != nil { + return "", fmt.Errorf("Process GO template failed with error: %v", err) + } + } + return result, nil +} + +// The actual template processing for Go templates +func (cf *ConfigFile) processGoTemplate(file string, configValues []byte) (string, error) { + + // The JSON configuration + var confData map[string]interface{} + var res bytes.Buffer + + err := encjson.Unmarshal(configValues, &confData) + utils.CheckErr(err) + + // Read the template + data, err := os.ReadFile(file) + utils.CheckErr(err) + + tpl, err := template.New("conf").Parse(string(data)) + utils.CheckErr(err) + + utils.CheckErr(tpl.Execute(&res, confData["Config"])) + + return res.String(), nil +} + +// The actual template processing for HCL templates +func (cf *ConfigFile) processHCLTemplate(file string, config []byte) (string, error) { + + fct, err := os.ReadFile(file) + utils.CheckErr(err) + + expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) + utils.CheckDiags(diags) + + // Retrieve values from JSON + var varsVal cty.Value + ctyType, err := ctyjson.ImpliedType(config) + if err != nil { + return "", err + /* Maybe one day + cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) + if diags.HasErrors() { + panic(diags.Error()) + } + varsVal, diags = cexpr.Value(&hcl.EvalContext{}) + fmt.Println(cexpr.Variables()) + checkDiags(diags) + */ + } else { + varsVal, err = ctyjson.Unmarshal(config, ctyType) + utils.CheckErr(err) + } + + ctx := &hcl.EvalContext{ + Variables: varsVal.AsValueMap(), + } + + for n := range ctx.Variables { + if !hclsyntax.ValidIdentifier(n) { + return "", fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n) + } + } + + for _, traversal := range expr.Variables() { + root := traversal.RootName() + if _, ok := ctx.Variables[root]; !ok { + return "", fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange()) + } + } + + val, diags := expr.Value(ctx) + if diags.HasErrors() { + return "", diags + } + + return val.AsString(), nil +} diff --git a/pkg/templater/main.go b/pkg/templater/main.go index f1fba18..8a62ee3 100644 --- a/pkg/templater/main.go +++ b/pkg/templater/main.go @@ -1,89 +1,88 @@ package templater import ( - "bytes" - encjson "encoding/json" "fmt" + "io/ioutil" + "log" "os" - "text/template" - "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/zclconf/go-cty/cty" - ctyjson "github.com/zclconf/go-cty/cty/json" - - "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" + "github.com/imdario/mergo" ) -func ProcessGoTemplate(file string, config []byte) string { +var CacheFilePath = "/var/cache/templater.db" - // The JSON configuration - var confData map[string]interface{} - var res bytes.Buffer - - err := encjson.Unmarshal(config, &confData) - utils.CheckErr(err) - - // Read the template - data, err := os.ReadFile(file) - utils.CheckErr(err) - - tpl, err := template.New("conf").Parse(string(data)) - utils.CheckErr(err) - - utils.CheckErr(tpl.Execute(&res, confData)) - - return res.String() +type TemplaterConfig struct { + Name string `json:"Name"` + TemplateDirectory string `json:"TemplateDirectory"` + Services map[string]Service `json:"Services"` + GlobalService Service `json:"Global"` } -func ProcessHCLTemplate(file string, config []byte) string { - - fct, err := os.ReadFile(file) - utils.CheckErr(err) - - expr, diags := hclsyntax.ParseTemplate(fct, file, hcl.Pos{Line: 0, Column: 1}) - utils.CheckDiags(diags) - - // Retrieve values from JSON - var varsVal cty.Value - ctyType, err := ctyjson.ImpliedType(config) +func (tc *TemplaterConfig) loadCache() error { + // Load globals from cache + var cache Service + err := Load(CacheFilePath, &cache) if err != nil { - panic(err) - /* Maybe one day - cexpr, diags := hclsyntax.ParseExpression(config, "", hcl.Pos{Line: 0, Column: 1}) - if diags.HasErrors() { - panic(diags.Error()) - } - varsVal, diags = cexpr.Value(&hcl.EvalContext{}) - fmt.Println(cexpr.Variables()) - checkDiags(diags) - */ - } else { - varsVal, err = ctyjson.Unmarshal(config, ctyType) - utils.CheckErr(err) + fmt.Printf("Warning: No globals to load\n") } - ctx := &hcl.EvalContext{ - Variables: varsVal.AsValueMap(), + err = mergo.Merge(&tc.GlobalService, cache) + if err != nil { + return err } - - for n := range ctx.Variables { - if !hclsyntax.ValidIdentifier(n) { - panic(fmt.Errorf("invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)) - } - } - - for _, traversal := range expr.Variables() { - root := traversal.RootName() - if _, ok := ctx.Variables[root]; !ok { - panic(fmt.Errorf("vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())) - } - } - - val, diags := expr.Value(ctx) - if diags.HasErrors() { - panic(diags.Error()) - } - - return val.AsString() + return nil +} + +func (tc *TemplaterConfig) New(confpath string, templateDir string) error { + // Load stored globals if needed + lerr := tc.loadCache() + if lerr != nil { + return lerr + } + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(confpath) + if err != nil { + return err + } + + if fileInfo.IsDir() { + // The conf path is a directory we load all the files and merge data + files, err := ioutil.ReadDir(confpath) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + for _, file := range files { + fname := fmt.Sprintf("%s/%s", confpath, file.Name()) + var ntc TemplaterConfig + err := Load(fname, &ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + err = mergo.Merge(tc, ntc) + if err != nil { + return fmt.Errorf("Templater configuration load failed with error: %v", err) + } + } + } else { + // The conf path is a file we only load this file (of course) + err = Load(confpath, tc) + if err != nil { + return fmt.Errorf("Confiuration read failed with error: %v", err) + } + } + + tc.TemplateDirectory = templateDir + + return nil +} + +func (tc *TemplaterConfig) ManageServices(rootDir string) error { + for name, svr := range tc.Services { + log.Printf("*** Working on service %s", name) + if err := svr.Manage(tc.TemplateDirectory, rootDir); err != nil { + return err + } + log.Printf("*** Service %s processed", name) + } + return nil } diff --git a/pkg/templater/packages.go b/pkg/templater/packages.go new file mode 100644 index 0000000..98dca8d --- /dev/null +++ b/pkg/templater/packages.go @@ -0,0 +1,74 @@ +package templater + +import ( + "fmt" + "log" + "runtime" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemPackage struct { + Name string `json:"name"` + Type string `json:"type"` + Action string `json:"action"` + OS string `json:"os"` + Distribution string `json:"distribution"` +} + +func (p *SystemPackage) SetDistribution() error { + OSConfig, err := utils.ReadOSRelease() + if err != nil { + return err + } + + p.Distribution = OSConfig["ID_LIKE"] + return nil +} +func (p *SystemPackage) SetOS() error { + p.OS = runtime.GOOS + return nil +} + +func (p *SystemPackage) Manage() error { + var pkErr error + var stdErr []byte + + if p.OS == "" { + if err := p.SetOS(); err != nil { + return err + } + } + + if p.Distribution == "" { + if err := p.SetDistribution(); err != nil { + return err + } + } + + log.Printf("\tInstalling %s package\n", p.Name) + switch os := p.Distribution; os { + case "debian", "ubuntu": + _, stdErr, pkErr = utils.RunSystemCommand("apt", "install", "-y", p.Name) + case "alpine": + _, stdErr, pkErr = utils.RunSystemCommand("apk", "add", p.Name) + case "redhat": + _, stdErr, pkErr = utils.RunSystemCommand("yum", "install", "-y", p.Name) + case "arch": + _, stdErr, pkErr = utils.RunSystemCommand("pacman", "-Suy", p.Name) + default: + pkErr = fmt.Errorf("Unsupported OS %s [%s]", p.OS, stdErr) + } + + if pkErr != nil { + var msg string + if len(stdErr) != 0 { + msg = string(stdErr) + } else { + msg = pkErr.Error() + } + return fmt.Errorf("Package %s, os %s, failed with error: %v", p.Name, p.OS, msg) + } + + return nil +} diff --git a/pkg/templater/persist.go b/pkg/templater/persist.go new file mode 100644 index 0000000..edc2155 --- /dev/null +++ b/pkg/templater/persist.go @@ -0,0 +1,51 @@ +package templater + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "sync" +) + +var lock sync.Mutex + +func marshal(v interface{}) (io.Reader, error) { + b, err := json.MarshalIndent(v, "", "\t") + if err != nil { + return nil, err + } + return bytes.NewReader(b), nil +} + +func unmarshal(r io.Reader, v interface{}) error { + return json.NewDecoder(r).Decode(v) +} + +func Save(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("Saving Templater configuration failed with error : %v", err) + } + defer f.Close() + r, err := marshal(v) + if err != nil { + return err + } + _, err = io.Copy(f, r) + return err +} + +func Load(path string, v interface{}) error { + lock.Lock() + defer lock.Unlock() + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + return unmarshal(f, v) +} diff --git a/pkg/templater/repo-apk.go b/pkg/templater/repo-apk.go new file mode 100644 index 0000000..c8f88a6 --- /dev/null +++ b/pkg/templater/repo-apk.go @@ -0,0 +1,102 @@ +package templater + +import ( + "bufio" + "fmt" + "io/ioutil" + "log" + "os" + "strings" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +var APKConfigFile = "/etc/apk/repositories" + +type APKRepository struct { + Repository +} + +func (hr *APKRepository) urlIsPresent() (bool, error) { + f, err := os.Open(APKConfigFile) + if err != nil { + return false, err + } + defer f.Close() + + // Splits on newlines by default. + scanner := bufio.NewScanner(f) + + line := 1 + for scanner.Scan() { + if strings.Contains(scanner.Text(), hr.URL) { + log.Printf("\tRepository %s already present\n", hr.Name) + return true, nil + } + line++ + } + + if err := scanner.Err(); err != nil { + return false, err + } + return false, nil + +} + +func (hr *APKRepository) Add() error { + + URLIsPresent, err := hr.urlIsPresent() + if err != nil { + return err + } + + if URLIsPresent { + return nil + } else { + data := fmt.Sprintf("%s\n", hr.URL) + file, err := os.OpenFile(APKConfigFile, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + if _, err := file.WriteString(data); err != nil { + return err + } else { + log.Printf("Repository %s added\n", hr.Name) + } + return nil + } + +} + +func (hr *APKRepository) Update() error { + if _, stdErr, err := utils.RunSystemCommand("apk", "update"); err != nil { + return fmt.Errorf("%s [%s]", stdErr, err) + } + return nil +} + +// FIXME +func (hr *APKRepository) Delete() error { + fileBytes, err := ioutil.ReadFile("/etc/apk/repositories") + if err != nil { + return err + } + lines := strings.Split(string(fileBytes), "\n") + for _, line := range lines { + fmt.Printf("DEBUG TODO %s", line) + } + return nil +} + +func (hr *APKRepository) Manage() error { + if hr.Enabled { + if err := hr.Add(); err != nil { + return err + } + log.Println("\tUpdating apk repositories") + return hr.Update() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo-deb.go b/pkg/templater/repo-deb.go new file mode 100644 index 0000000..1c6f6f9 --- /dev/null +++ b/pkg/templater/repo-deb.go @@ -0,0 +1,43 @@ +package templater + +import ( + "fmt" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type DebRepository struct { + Repository +} + +func (hr *DebRepository) Add() error { + //deb http://fr.archive.ubuntu.com/ubuntu/ focal main restricted + + data := fmt.Sprintf("deb %s", hr.URL) + if err := os.WriteFile("/etc/apt/source.list.d", []byte(data), 0600); err != nil { + return err + } + + return nil +} + +func (hr *DebRepository) Update() error { + if _, stdErr, err := utils.RunSystemCommand("apt", "update", "-y"); err != nil { + return fmt.Errorf("%s [%s]", stdErr, err) + } + + return nil +} + +func (hr *DebRepository) Delete() error { + //TODO + return nil +} +func (hr *DebRepository) Manage() error { + if hr.Enabled { + return hr.Add() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo-helm.go b/pkg/templater/repo-helm.go new file mode 100644 index 0000000..71f1803 --- /dev/null +++ b/pkg/templater/repo-helm.go @@ -0,0 +1,25 @@ +package templater + +type HelmRepository struct { + Repository +} + +func (hr *HelmRepository) Add() error { + return nil +} + +func (hr *HelmRepository) Update() error { + return nil +} + +func (hr *HelmRepository) Delete() error { + return nil +} + +func (hr *HelmRepository) Manage() error { + if hr.Enabled { + return hr.Add() + } else { + return hr.Delete() + } +} diff --git a/pkg/templater/repo.go b/pkg/templater/repo.go new file mode 100644 index 0000000..8c143d0 --- /dev/null +++ b/pkg/templater/repo.go @@ -0,0 +1,17 @@ +package templater + +type PackageRepository interface { + Manage() error + Update() error + Add() error + Delete() error +} + +type Repository struct { + Actions PackageRepository + + Name string `json:"name"` + Type string `json:"type"` + URL string `json:"url"` + Enabled bool `json:"enabled"` +} diff --git a/pkg/templater/services.go b/pkg/templater/services.go new file mode 100644 index 0000000..b95779d --- /dev/null +++ b/pkg/templater/services.go @@ -0,0 +1,122 @@ +package templater + +import ( + "encoding/json" + "fmt" + "log" + "path/filepath" +) + +type Service struct { + ConfigFiles []ConfigFile `json:"ConfigFiles"` + Vars map[string]interface{} `json:"Vars"` + Daemons map[string]SystemService `json:"Daemons"` + Users map[string]SystemUser `json:"Users"` + Repos map[string]Repository `json:"Repositories"` + Packages map[string]SystemPackage `json:"Packages"` +} + +func (s *Service) manageRepos(repos map[string]Repository) error { + for _, repo := range s.Repos { + if repo.Type == "helm" { + rp := HelmRepository{repo} + if err := rp.Manage(); err != nil { + return err + } + } + if repo.Type == "apk" { + rp := APKRepository{repo} + if err := rp.Manage(); err != nil { + return err + } + } + if repo.Type == "deb" { + rp := DebRepository{} + if err := rp.Manage(); err != nil { + return err + } + } + + } + return nil +} + +func (s *Service) Manage(templateDir string, rootDir string) error { + // Manage packages repositories + log.Print(" Managing package repositories") + err := s.manageRepos(s.Repos) + if err != nil { + return err + } + + // Create system users + log.Print(" Managing system users") + for _, user := range s.Users { + err := user.Manage() + if err != nil { + return err + } + } + + // Manage system packages + log.Print(" Installing packages") + for _, pack := range s.Packages { + err := pack.Manage() + if err != nil { + return err + } + log.Printf("\tPackage %s installed\n", pack.Name) + } + + log.Print(" Generating configuration files\n") + err = processConfigFiles(s, templateDir, rootDir) + if err != nil { + return fmt.Errorf("ProcessingTemplatesFailed with error: %v", err) + } + + log.Print(" Managing services:\n") + for _, daemon := range s.Daemons { + err = daemon.Manage() + if err != nil { + return fmt.Errorf("Error managing service daemons: %v", err) + } + } + return nil +} + +func processConfigFiles(s *Service, templateDir string, rootDir string) error { + values, err := json.Marshal(s) + if err != nil { + return fmt.Errorf("Error unmarshaling values on template process; %v", err) + } + + var servicesToRestart []string + for _, tpl := range s.ConfigFiles { + fileExt := filepath.Ext(tpl.Source) + if fileExt == ".hcl" { + tpl.TemplateType = "hcl" + } else if fileExt == ".tpl" { + tpl.TemplateType = "go" + } else { + return fmt.Errorf("Unsupported file type %s, templates extensions have to be '.hcl' or '.tpl'", fileExt) + } + if err := tpl.Generate(rootDir, templateDir, values); err != nil { + return fmt.Errorf("Template %s generation failed with error %v", tpl.Source, err) + } + + if len(tpl.Service) != 0 { + servicesToRestart = append(servicesToRestart, tpl.Service) + } + } + + for _, srv := range servicesToRestart { + sv := SystemService{ + Name: srv, + Enabled: true, + Type: "", + ToStart: true, + } + return sv.Restart() + } + return nil +} diff --git a/pkg/templater/system_services.go b/pkg/templater/system_services.go new file mode 100644 index 0000000..582cb2c --- /dev/null +++ b/pkg/templater/system_services.go @@ -0,0 +1,132 @@ +package templater + +import ( + "fmt" + "log" + "os" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemService struct { + Name string `json:"name"` + Enabled bool `json:"enabled"` + Type string `json:"type"` + ToStart bool `json:"start"` +} + +func (sys *SystemService) SetType() { + systemdRunDirectory := "/run/systemd/system" + openRcBinaryFile := "/sbin/openrc" + + // Check if the configuration path is a Directory or a file + fileInfo, err := os.Stat(systemdRunDirectory) + if err == nil { + if fileInfo.IsDir() { + sys.Type = "systemd" + } + } + + fileInfo, err = os.Stat(openRcBinaryFile) + if err == nil { + if fileInfo.IsDir() { + return + } + sys.Type = "openrc" + } +} + +func (sys *SystemService) Action() error { + if sys.ToStart { + return sys.Start() + } + return nil +} + +func (sys *SystemService) Manage() error { + // By default if the property sys.ToStart is empty + if sys.Type == "" || sys.Type == "auto" { + sys.SetType() + } + if sys.Enabled { + err := sys.Enable() + if err != nil { + return err + } + if err = sys.Action(); err != nil { + return err + } + + } else { + log.Printf("\nNothing to do for daemon %s\n", sys.Name) + } + return nil +} + +func (sys *SystemService) Start() error { + log.Printf("\tStarting system service : %s\n", sys.Name) + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "start", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Start error:\n - %s", sys.Name, stdErr) + } + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("service", sys.Name, "start") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} + +func (sys *SystemService) Stop() error { + log.Printf("\tStopping system service : %s\n", sys.Name) + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "stop", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Stop error:\n - %s", sys.Name, stdErr) + } + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("service", sys.Name, "stop") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} + +func (sys *SystemService) Restart() error { + if sys.Type == "" || sys.Type == "auto" { + sys.SetType() + } + if err := sys.Stop(); err != nil { + return err + } + if err := sys.Start(); err != nil { + return err + } + return nil +} + +func (sys *SystemService) Enable() error { + if sys.Type == "systemd" { + _, stdErr, err := utils.RunSystemCommand("systemctl", "enable", sys.Name) + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + log.Printf("\tSystemd service %s enabled", sys.Name) + } else if sys.Type == "openrc" { + _, stdErr, err := utils.RunSystemCommand("rc-update", "add", sys.Name, "default") + if err != nil { + return fmt.Errorf("System service %s \n * Enable error:\n - %s", sys.Name, stdErr) + } + log.Printf("\tOpenRC service %s enabled", sys.Name) + } else { + return fmt.Errorf("Unsupported service type %s for service %s", sys.Type, sys.Name) + } + return nil +} diff --git a/pkg/templater/system_users.go b/pkg/templater/system_users.go new file mode 100644 index 0000000..bd870a9 --- /dev/null +++ b/pkg/templater/system_users.go @@ -0,0 +1,51 @@ +package templater + +import ( + "log" + + "forge.cadoles.com/pcaseiro/templatefile/pkg/utils" +) + +type SystemUser struct { + UserName string `json:"username"` + Group string `json:"group"` + Home string `json:"home"` + Shell string `json:"shell"` +} + +func (su *SystemUser) exists() (bool, error) { + _, _, err := utils.RunSystemCommand("getent", "passwd", su.UserName) + if err != nil { + return false, err + } + return true, nil +} + +func (su *SystemUser) Manage() error { + exist, _ := su.exists() + if exist { + log.Printf("\tUser %s already exists", su.UserName) + return nil + } + return su.Create() +} + +func (su *SystemUser) Create() error { + _, _, err := utils.RunSystemCommand("useradd", "-b", su.Home, "-m", "-G", su.Group, su.UserName) + if err != nil { + return err + } + return nil +} + +func (su *SystemUser) Delete() error { + _, _, err := utils.RunSystemCommand("userdel", su.UserName) + if err != nil { + return err + } + return nil +} + +func (su *SystemUser) Update() error { + return nil +} diff --git a/pkg/utils/main.go b/pkg/utils/main.go index 8ea276d..2c92b51 100644 --- a/pkg/utils/main.go +++ b/pkg/utils/main.go @@ -1,6 +1,9 @@ package utils import ( + "bytes" + "os/exec" + "github.com/hashicorp/hcl/v2" ) @@ -15,3 +18,15 @@ func CheckDiags(diag hcl.Diagnostics) { panic(diag.Error()) } } + +// Execute a system command ... +func RunSystemCommand(name string, arg ...string) ([]byte, []byte, error) { + var stdOut bytes.Buffer + var stdErr bytes.Buffer + + cmd := exec.Command(name, arg...) + cmd.Stderr = &stdErr + cmd.Stdout = &stdOut + err := cmd.Run() + return stdOut.Bytes(), stdErr.Bytes(), err +} diff --git a/pkg/utils/os.go b/pkg/utils/os.go new file mode 100644 index 0000000..8fe4d64 --- /dev/null +++ b/pkg/utils/os.go @@ -0,0 +1,27 @@ +package utils + +import ( + "fmt" + + ini "gopkg.in/ini.v1" +) + +var osReleaseFile = "/etc/os-release" + +func ReadOSRelease() (map[string]string, error) { + cfg, err := ini.Load(osReleaseFile) + if err != nil { + return nil, fmt.Errorf("Fail to read file: %v ", err) + } + + ConfigParams := make(map[string]string) + ConfigParams["ID"] = cfg.Section("").Key("ID").String() + idLike := cfg.Section("").Key("ID_LIKE").String() + if idLike != "" { + ConfigParams["ID_LIKE"] = idLike + } else { + ConfigParams["ID_LIKE"] = ConfigParams["ID"] + } + + return ConfigParams, nil +}