diff --git a/Makefile b/Makefile index d523831..622e8b3 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ clean: rm -f emissary.sqlite* rm -f server-key.json rm -f agent-key.json + rm -f state.json .PHONY: test test: test-go ## Executing tests diff --git a/internal/agent/controller/app/controller.go b/internal/agent/controller/app/controller.go index b87af79..3bce0fb 100644 --- a/internal/agent/controller/app/controller.go +++ b/internal/agent/controller/app/controller.go @@ -38,7 +38,7 @@ func (c *Controller) Name() string { func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { appSpec := spec.NewSpec() - if err := state.GetSpec(spec.Name, appSpec); err != nil { + if err := state.GetSpec(spec.Name, spec.Version, appSpec); err != nil { if errors.Is(err, agent.ErrSpecNotFound) { logger.Info(ctx, "could not find app spec") @@ -50,7 +50,12 @@ func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { return errors.WithStack(err) } - logger.Info(ctx, "retrieved spec", logger.F("spec", appSpec.SpecName()), logger.F("revision", appSpec.SpecRevision())) + logger.Info( + ctx, "retrieved spec", + logger.F("name", appSpec.SpecDefinitionName()), + logger.F("version", appSpec.SpecDefinitionVersion()), + logger.F("revision", appSpec.SpecRevision()), + ) c.updateApps(ctx, appSpec) diff --git a/internal/agent/controller/app/spec/init.go b/internal/agent/controller/app/spec/init.go index e6983ae..75d23fa 100644 --- a/internal/agent/controller/app/spec/init.go +++ b/internal/agent/controller/app/spec/init.go @@ -11,7 +11,7 @@ import ( var schema []byte func init() { - if err := spec.Register(Name, schema); err != nil { + if err := spec.Register(string(Name), Version, schema); err != nil { panic(errors.WithStack(err)) } } diff --git a/internal/agent/controller/app/spec/schema.json b/internal/agent/controller/app/spec/schema.json index d293026..cb14da4 100644 --- a/internal/agent/controller/app/spec/schema.json +++ b/internal/agent/controller/app/spec/schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://app.edge.emissary.cadoles.com/spec.json", "title": "AppSpec", "description": "Emissary 'App' specification", @@ -78,7 +78,9 @@ "type": "string" } }, - "required": ["defaultUrlTemplate"], + "required": [ + "defaultUrlTemplate" + ], "additionalProperties": false }, "unexpectedHostRedirect": { @@ -94,7 +96,10 @@ "type": "string" } }, - "required": ["acceptedHostPatterns", "hostTarget"], + "required": [ + "acceptedHostPatterns", + "hostTarget" + ], "additionalProperties": false }, "auth": { @@ -104,7 +109,10 @@ "type": "object", "properties": { "key": { - "type": ["object", "string"] + "type": [ + "object", + "string" + ] }, "signingAlgorithm": { "type": "string" diff --git a/internal/agent/controller/app/spec/spec.go b/internal/agent/controller/app/spec/spec.go index d5c8af4..f029416 100644 --- a/internal/agent/controller/app/spec/spec.go +++ b/internal/agent/controller/app/spec/spec.go @@ -6,7 +6,10 @@ import ( "github.com/lestrrat-go/jwx/v2/jwa" ) -const Name spec.Name = "app.emissary.cadoles.com" +const ( + Name string = "app.emissary.cadoles.com" + Version string = "0.0.0" +) type Spec struct { Revision int `json:"revision"` @@ -56,10 +59,14 @@ type AppURLResolving struct { DefaultURLTemplate string `json:"defaultUrlTemplate"` } -func (s *Spec) SpecName() spec.Name { +func (s *Spec) SpecDefinitionName() string { return Name } +func (s *Spec) SpecDefinitionVersion() string { + return Version +} + func (s *Spec) SpecRevision() int { return s.Revision } diff --git a/internal/agent/controller/app/spec/validator_test.go b/internal/agent/controller/app/spec/validator_test.go index b360f65..2acc659 100644 --- a/internal/agent/controller/app/spec/validator_test.go +++ b/internal/agent/controller/app/spec/validator_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "testing" + "forge.cadoles.com/Cadoles/emissary/internal/datastore/memory" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) @@ -27,11 +28,15 @@ var validatorTestCases = []validatorTestCase{ func TestValidator(t *testing.T) { t.Parallel() - validator := spec.NewValidator() - if err := validator.Register(Name, schema); err != nil { + ctx := context.Background() + + repo := memory.NewSpecDefinitionRepository() + if _, err := repo.Upsert(ctx, Name, Version, schema); err != nil { t.Fatalf("+%v", errors.WithStack(err)) } + validator := spec.NewValidator(repo) + for _, tc := range validatorTestCases { func(tc validatorTestCase) { t.Run(tc.Name, func(t *testing.T) { diff --git a/internal/agent/controller/mdns/controller.go b/internal/agent/controller/mdns/controller.go index 0f1e7c5..8daca1b 100644 --- a/internal/agent/controller/mdns/controller.go +++ b/internal/agent/controller/mdns/controller.go @@ -33,7 +33,7 @@ func (c *Controller) Name() string { func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { mdnsSpec := mdns.NewSpec() - if err := state.GetSpec(mdns.Name, mdnsSpec); err != nil { + if err := state.GetSpec(mdns.Name, mdns.Version, mdnsSpec); err != nil { if errors.Is(err, agent.ErrSpecNotFound) { logger.Info(ctx, "could not find mdns spec") @@ -45,7 +45,11 @@ func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { return errors.WithStack(err) } - logger.Info(ctx, "retrieved spec", logger.F("spec", mdnsSpec.SpecName()), logger.F("revision", mdnsSpec.SpecRevision())) + logger.Info(ctx, "retrieved spec", + logger.F("name", mdnsSpec.SpecDefinitionName()), + logger.F("version", mdnsSpec.SpecDefinitionVersion()), + logger.F("revision", mdnsSpec.SpecRevision()), + ) if err := c.updateResponder(ctx, mdnsSpec); err != nil { return errors.Wrap(err, "could not update responder") diff --git a/internal/agent/controller/mdns/spec/init.go b/internal/agent/controller/mdns/spec/init.go index e6983ae..75d23fa 100644 --- a/internal/agent/controller/mdns/spec/init.go +++ b/internal/agent/controller/mdns/spec/init.go @@ -11,7 +11,7 @@ import ( var schema []byte func init() { - if err := spec.Register(Name, schema); err != nil { + if err := spec.Register(string(Name), Version, schema); err != nil { panic(errors.WithStack(err)) } } diff --git a/internal/agent/controller/mdns/spec/schema.json b/internal/agent/controller/mdns/spec/schema.json index 78bb910..555fcf2 100644 --- a/internal/agent/controller/mdns/spec/schema.json +++ b/internal/agent/controller/mdns/spec/schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://mdns.edge.emissary.cadoles.com/spec.json", "title": "MDNSSpec", "description": "Emissary 'MDNS' specification", diff --git a/internal/agent/controller/mdns/spec/spec.go b/internal/agent/controller/mdns/spec/spec.go index 0ff32ff..b4062ee 100644 --- a/internal/agent/controller/mdns/spec/spec.go +++ b/internal/agent/controller/mdns/spec/spec.go @@ -4,7 +4,10 @@ import ( "forge.cadoles.com/Cadoles/emissary/internal/spec" ) -const Name spec.Name = "mdns.emissary.cadoles.com" +const ( + Name string = "mdns.emissary.cadoles.com" + Version string = "0.0.0" +) type Spec struct { Revision int `json:"revision"` @@ -19,10 +22,14 @@ type Service struct { Port int `json:"port"` } -func (s *Spec) SpecName() spec.Name { +func (s *Spec) SpecDefinitionName() string { return Name } +func (s *Spec) SpecDefinitionVersion() string { + return Version +} + func (s *Spec) SpecRevision() int { return s.Revision } diff --git a/internal/agent/controller/mdns/spec/validator_test.go b/internal/agent/controller/mdns/spec/validator_test.go index b360f65..2acc659 100644 --- a/internal/agent/controller/mdns/spec/validator_test.go +++ b/internal/agent/controller/mdns/spec/validator_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "testing" + "forge.cadoles.com/Cadoles/emissary/internal/datastore/memory" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) @@ -27,11 +28,15 @@ var validatorTestCases = []validatorTestCase{ func TestValidator(t *testing.T) { t.Parallel() - validator := spec.NewValidator() - if err := validator.Register(Name, schema); err != nil { + ctx := context.Background() + + repo := memory.NewSpecDefinitionRepository() + if _, err := repo.Upsert(ctx, Name, Version, schema); err != nil { t.Fatalf("+%v", errors.WithStack(err)) } + validator := spec.NewValidator(repo) + for _, tc := range validatorTestCases { func(tc validatorTestCase) { t.Run(tc.Name, func(t *testing.T) { diff --git a/internal/agent/controller/openwrt/spec/sysupgrade/init.go b/internal/agent/controller/openwrt/spec/sysupgrade/init.go index f3d3b71..d36c670 100644 --- a/internal/agent/controller/openwrt/spec/sysupgrade/init.go +++ b/internal/agent/controller/openwrt/spec/sysupgrade/init.go @@ -11,7 +11,7 @@ import ( var schema []byte func init() { - if err := spec.Register(Name, schema); err != nil { + if err := spec.Register(string(Name), Version, schema); err != nil { panic(errors.WithStack(err)) } } diff --git a/internal/agent/controller/openwrt/spec/sysupgrade/schema.json b/internal/agent/controller/openwrt/spec/sysupgrade/schema.json index eef8941..141ac62 100644 --- a/internal/agent/controller/openwrt/spec/sysupgrade/schema.json +++ b/internal/agent/controller/openwrt/spec/sysupgrade/schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://sysupgrade.openwrt.emissary.cadoles.com/spec.json", "title": "SysUpgradeSpec", "description": "Emissary 'SysUpgrade' specification", @@ -15,6 +15,10 @@ "type": "string" } }, - "required": ["url", "sha256sum", "version"], + "required": [ + "url", + "sha256sum", + "version" + ], "additionalProperties": false } \ No newline at end of file diff --git a/internal/agent/controller/openwrt/spec/sysupgrade/spec.go b/internal/agent/controller/openwrt/spec/sysupgrade/spec.go index ca0559d..7ac228a 100644 --- a/internal/agent/controller/openwrt/spec/sysupgrade/spec.go +++ b/internal/agent/controller/openwrt/spec/sysupgrade/spec.go @@ -4,7 +4,10 @@ import ( "forge.cadoles.com/Cadoles/emissary/internal/spec" ) -const Name spec.Name = "sysupgrade.openwrt.emissary.cadoles.com" +const ( + Name string = "sysupgrade.openwrt.emissary.cadoles.com" + Version string = "0.0.0" +) type Spec struct { Revision int `json:"revision"` @@ -13,10 +16,14 @@ type Spec struct { Version string `json:"version"` } -func (s *Spec) SpecName() spec.Name { +func (s *Spec) SpecDefinitionName() string { return Name } +func (s *Spec) SpecDefinitionVersion() string { + return Version +} + func (s *Spec) SpecRevision() int { return s.Revision } diff --git a/internal/agent/controller/openwrt/spec/sysupgrade/validator_test.go b/internal/agent/controller/openwrt/spec/sysupgrade/validator_test.go index 7ce6e88..8e5fe09 100644 --- a/internal/agent/controller/openwrt/spec/sysupgrade/validator_test.go +++ b/internal/agent/controller/openwrt/spec/sysupgrade/validator_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "testing" + "forge.cadoles.com/Cadoles/emissary/internal/datastore/memory" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) @@ -27,11 +28,15 @@ var validatorTestCases = []validatorTestCase{ func TestValidator(t *testing.T) { t.Parallel() - validator := spec.NewValidator() - if err := validator.Register(Name, schema); err != nil { + ctx := context.Background() + + repo := memory.NewSpecDefinitionRepository() + if _, err := repo.Upsert(ctx, Name, Version, schema); err != nil { t.Fatalf("+%v", errors.WithStack(err)) } + validator := spec.NewValidator(repo) + for _, tc := range validatorTestCases { func(tc validatorTestCase) { t.Run(tc.Name, func(t *testing.T) { diff --git a/internal/agent/controller/openwrt/sysupgrade_controller.go b/internal/agent/controller/openwrt/sysupgrade_controller.go index ef82443..963812c 100644 --- a/internal/agent/controller/openwrt/sysupgrade_controller.go +++ b/internal/agent/controller/openwrt/sysupgrade_controller.go @@ -31,7 +31,7 @@ func (*SysUpgradeController) Name() string { func (c *SysUpgradeController) Reconcile(ctx context.Context, state *agent.State) error { sysSpec := sysupgrade.NewSpec() - if err := state.GetSpec(sysupgrade.Name, sysSpec); err != nil { + if err := state.GetSpec(sysupgrade.Name, sysupgrade.Version, sysSpec); err != nil { if errors.Is(err, agent.ErrSpecNotFound) { logger.Info(ctx, "could not find sysupgrade spec, doing nothing") diff --git a/internal/agent/controller/openwrt/uci_controller.go b/internal/agent/controller/openwrt/uci_controller.go index d62bf45..380b640 100644 --- a/internal/agent/controller/openwrt/uci_controller.go +++ b/internal/agent/controller/openwrt/uci_controller.go @@ -27,7 +27,7 @@ func (*UCIController) Name() string { func (c *UCIController) Reconcile(ctx context.Context, state *agent.State) error { uciSpec := ucispec.NewSpec() - if err := state.GetSpec(ucispec.NameUCI, uciSpec); err != nil { + if err := state.GetSpec(ucispec.Name, ucispec.Version, uciSpec); err != nil { if errors.Is(err, agent.ErrSpecNotFound) { logger.Info(ctx, "could not find uci spec, doing nothing") @@ -37,7 +37,11 @@ func (c *UCIController) Reconcile(ctx context.Context, state *agent.State) error return errors.WithStack(err) } - logger.Info(ctx, "retrieved spec", logger.F("spec", uciSpec.SpecName()), logger.F("revision", uciSpec.SpecRevision())) + logger.Info(ctx, "retrieved spec", + logger.F("name", uciSpec.SpecDefinitionName()), + logger.F("version", uciSpec.SpecDefinitionVersion()), + logger.F("revision", uciSpec.SpecRevision()), + ) if c.currentSpecRevision == uciSpec.SpecRevision() { logger.Info(ctx, "spec revision did not change, doing nothing") diff --git a/internal/agent/controller/persistence/controller.go b/internal/agent/controller/persistence/controller.go index a77ba1e..20d0f60 100644 --- a/internal/agent/controller/persistence/controller.go +++ b/internal/agent/controller/persistence/controller.go @@ -9,13 +9,12 @@ import ( "path/filepath" "forge.cadoles.com/Cadoles/emissary/internal/agent" - "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" "gitlab.com/wpetit/goweb/logger" ) type Controller struct { - trackedSpecRevisions map[spec.Name]int + trackedSpecRevisions map[string]map[string]int filename string loaded bool } @@ -78,36 +77,39 @@ func (c *Controller) specChanged(specs agent.Specs) bool { return true } - for name, spec := range specs { - trackedRevision, exists := c.trackedSpecRevisions[name] + for name, specVersions := range specs { + trackedSpecs, exists := c.trackedSpecRevisions[name] if !exists { return true } - if trackedRevision != spec.SpecRevision() { - return true - } - } + for version, spec := range specVersions { + trackedRevision, exists := trackedSpecs[version] + if !exists { + return true + } - for trackedSpecName, trackedRevision := range c.trackedSpecRevisions { - spec, exists := specs[trackedSpecName] - if !exists { - return true + if trackedRevision != spec.SpecRevision() { + return true + } } - if trackedRevision != spec.SpecRevision() { - return true - } } return false } func (c *Controller) trackSpecsRevisions(specs agent.Specs) { - c.trackedSpecRevisions = make(map[spec.Name]int) + c.trackedSpecRevisions = make(map[string]map[string]int) - for name, spec := range specs { - c.trackedSpecRevisions[name] = spec.SpecRevision() + for name, specVersions := range specs { + if _, exists := c.trackedSpecRevisions[name]; !exists { + c.trackedSpecRevisions[name] = make(map[string]int) + } + + for version, spec := range specVersions { + c.trackedSpecRevisions[name][version] = spec.SpecRevision() + } } } @@ -167,7 +169,7 @@ func (c *Controller) writeState(ctx context.Context, state *agent.State) error { } name := f.Name() - if err := ioutil.WriteFile(name, data, os.ModePerm); err != nil { + if err := os.WriteFile(name, data, os.ModePerm); err != nil { return errors.Errorf("cannot write data to temporary file %q: %v", name, err) } @@ -213,7 +215,7 @@ func (c *Controller) writeState(ctx context.Context, state *agent.State) error { func NewController(filename string) *Controller { return &Controller{ filename: filename, - trackedSpecRevisions: make(map[spec.Name]int), + trackedSpecRevisions: make(map[string]map[string]int), } } diff --git a/internal/agent/controller/proxy/controller.go b/internal/agent/controller/proxy/controller.go index 0ecdf70..ce55982 100644 --- a/internal/agent/controller/proxy/controller.go +++ b/internal/agent/controller/proxy/controller.go @@ -30,7 +30,7 @@ func (c *Controller) Name() string { func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { proxySpec := spec.NewSpec() - if err := state.GetSpec(spec.NameProxy, proxySpec); err != nil { + if err := state.GetSpec(spec.Name, spec.Version, proxySpec); err != nil { if errors.Is(err, agent.ErrSpecNotFound) { logger.Info(ctx, "could not find proxy spec") @@ -42,7 +42,12 @@ func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error { return errors.WithStack(err) } - logger.Info(ctx, "retrieved spec", logger.F("spec", proxySpec.SpecName()), logger.F("revision", proxySpec.SpecRevision())) + logger.Info( + ctx, "retrieved spec", + logger.F("name", proxySpec.SpecDefinitionName()), + logger.F("version", proxySpec.SpecDefinitionVersion()), + logger.F("revision", proxySpec.SpecRevision()), + ) c.updateProxies(ctx, proxySpec) diff --git a/internal/agent/state.go b/internal/agent/state.go index a27eaa3..df61cf4 100644 --- a/internal/agent/state.go +++ b/internal/agent/state.go @@ -11,7 +11,7 @@ import ( var ErrSpecNotFound = errors.New("spec not found") -type Specs map[spec.Name]spec.Spec +type Specs map[string]map[string]spec.Spec type State struct { agentID datastore.AgentID @@ -20,25 +20,33 @@ type State struct { func NewState() *State { return &State{ - specs: make(map[spec.Name]spec.Spec), + specs: make(map[string]map[string]spec.Spec), } } func (s *State) MarshalJSON() ([]byte, error) { state := struct { - ID datastore.AgentID `json:"agentId"` - Specs map[spec.Name]*spec.RawSpec `json:"specs"` + ID datastore.AgentID `json:"agentId"` + Specs map[string]map[string]*spec.RawSpec `json:"specs"` }{ ID: s.agentID, - Specs: func(specs map[spec.Name]spec.Spec) map[spec.Name]*spec.RawSpec { - rawSpecs := make(map[spec.Name]*spec.RawSpec) + Specs: func(specs map[string]map[string]spec.Spec) map[string]map[string]*spec.RawSpec { + rawSpecs := make(map[string]map[string]*spec.RawSpec) - for name, sp := range specs { - rawSpecs[name] = &spec.RawSpec{ - Name: sp.SpecName(), - Revision: sp.SpecRevision(), - Data: sp.SpecData(), + for name, versions := range specs { + if _, exists := rawSpecs[name]; !exists { + rawSpecs[name] = make(map[string]*spec.RawSpec) } + + for version, sp := range versions { + rawSpecs[name][version] = &spec.RawSpec{ + DefinitionName: sp.SpecDefinitionName(), + DefinitionVersion: sp.SpecDefinitionVersion(), + Revision: sp.SpecRevision(), + Data: sp.SpecData(), + } + } + } return rawSpecs @@ -55,19 +63,24 @@ func (s *State) MarshalJSON() ([]byte, error) { func (s *State) UnmarshalJSON(data []byte) error { state := struct { - AgentID datastore.AgentID `json:"agentId"` - Specs map[spec.Name]*spec.RawSpec `json:"specs"` + AgentID datastore.AgentID `json:"agentId"` + Specs map[string]map[string]*spec.RawSpec `json:"specs"` }{} if err := json.Unmarshal(data, &state); err != nil { return errors.WithStack(err) } - s.specs = func(rawSpecs map[spec.Name]*spec.RawSpec) Specs { + s.specs = func(rawSpecs map[string]map[string]*spec.RawSpec) Specs { specs := make(Specs) - for name, raw := range rawSpecs { - specs[name] = spec.Spec(raw) + for name, versions := range rawSpecs { + if _, exists := specs[name]; !exists { + specs[name] = make(map[string]spec.Spec) + } + for version, raw := range versions { + specs[name][version] = spec.Spec(raw) + } } return specs @@ -85,23 +98,36 @@ func (s *State) Specs() Specs { } func (s *State) ClearSpecs() *State { - s.specs = make(map[spec.Name]spec.Spec) + s.specs = make(map[string]map[string]spec.Spec) return s } func (s *State) SetSpec(sp spec.Spec) *State { if s.specs == nil { - s.specs = make(map[spec.Name]spec.Spec) + s.specs = make(map[string]map[string]spec.Spec) } - s.specs[sp.SpecName()] = sp + name := sp.SpecDefinitionName() + + if _, exists := s.specs[name]; !exists { + s.specs[name] = make(map[string]spec.Spec) + } + + version := sp.SpecDefinitionVersion() + + s.specs[name][version] = sp return s } -func (s *State) GetSpec(name spec.Name, dest any) error { - spec, exists := s.specs[name] +func (s *State) GetSpec(name string, version string, dest any) error { + versions, exists := s.specs[name] + if !exists { + return errors.WithStack(ErrSpecNotFound) + } + + spec, exists := versions[version] if !exists { return errors.WithStack(ErrSpecNotFound) } diff --git a/internal/command/client/agent/spec/delete.go b/internal/command/client/agent/spec/delete.go index 45d6cac..3d6292d 100644 --- a/internal/command/client/agent/spec/delete.go +++ b/internal/command/client/agent/spec/delete.go @@ -6,7 +6,6 @@ import ( agentFlag "forge.cadoles.com/Cadoles/emissary/internal/command/client/agent/flag" "forge.cadoles.com/Cadoles/emissary/internal/command/client/apierr" clientFlag "forge.cadoles.com/Cadoles/emissary/internal/command/client/flag" - "forge.cadoles.com/Cadoles/emissary/internal/spec" "forge.cadoles.com/Cadoles/emissary/pkg/client" "github.com/pkg/errors" "github.com/urfave/cli/v2" @@ -23,6 +22,11 @@ func DeleteCommand() *cli.Command { Name: "spec-name", Usage: "use `NAME` as specification's name", }, + &cli.StringFlag{ + Name: "spec-version", + Usage: "use `VERSION` as specification's version", + Value: "0.0.0", + }, ), Action: func(ctx *cli.Context) error { baseFlags := clientFlag.GetBaseFlags(ctx) @@ -37,14 +41,19 @@ func DeleteCommand() *cli.Command { return errors.WithStack(err) } - specName, err := assertSpecName(ctx) + specDefName, err := assertSpecDefName(ctx) + if err != nil { + return errors.WithStack(err) + } + + specDefVersion, err := assertSpecDefVersion(ctx) if err != nil { return errors.WithStack(err) } client := client.New(baseFlags.ServerURL, client.WithToken(token)) - specName, err = client.DeleteAgentSpec(ctx.Context, agentID, specName) + specDefName, specDefVersion, err = client.DeleteAgentSpec(ctx.Context, agentID, specDefName, specDefVersion) if err != nil { return errors.WithStack(apierr.Wrap(err)) } @@ -54,9 +63,11 @@ func DeleteCommand() *cli.Command { } if err := format.Write(baseFlags.Format, os.Stdout, hints, struct { - Name spec.Name `json:"name"` + Name string `json:"name"` + Version string `json:"version"` }{ - Name: specName, + Name: specDefName, + Version: specDefVersion, }); err != nil { return errors.WithStack(err) } diff --git a/internal/command/client/agent/spec/get.go b/internal/command/client/agent/spec/get.go index ea671cb..7dc0aa0 100644 --- a/internal/command/client/agent/spec/get.go +++ b/internal/command/client/agent/spec/get.go @@ -36,9 +36,7 @@ func GetCommand() *cli.Command { return errors.WithStack(apierr.Wrap(err)) } - hints := format.Hints{ - OutputMode: baseFlags.OutputMode, - } + hints := specHints(baseFlags.OutputMode) if err := format.Write(baseFlags.Format, os.Stdout, hints, clientFlag.AsAnySlice(specs)...); err != nil { return errors.WithStack(err) diff --git a/internal/command/client/agent/spec/hints.go b/internal/command/client/agent/spec/hints.go new file mode 100644 index 0000000..3b47ef9 --- /dev/null +++ b/internal/command/client/agent/spec/hints.go @@ -0,0 +1,21 @@ +package spec + +import ( + "gitlab.com/wpetit/goweb/cli/format" + "gitlab.com/wpetit/goweb/cli/format/table" +) + +func specHints(outputMode format.OutputMode) format.Hints { + return format.Hints{ + OutputMode: outputMode, + Props: []format.Prop{ + format.NewProp("ID", "ID"), + format.NewProp("Revision", "Revision"), + format.NewProp("DefinitionName", "Def. Name"), + format.NewProp("DefinitionVersion", "Def. Version"), + format.NewProp("Data", "Data"), + format.NewProp("CreatedAt", "CreatedAt", table.WithCompactModeMaxColumnWidth(20)), + format.NewProp("UpdatedAt", "UpdatedAt", table.WithCompactModeMaxColumnWidth(20)), + }, + } +} diff --git a/internal/command/client/agent/spec/update.go b/internal/command/client/agent/spec/update.go index 23da653..026cc57 100644 --- a/internal/command/client/agent/spec/update.go +++ b/internal/command/client/agent/spec/update.go @@ -24,6 +24,11 @@ func UpdateCommand() *cli.Command { Name: "spec-name", Usage: "use `NAME` as specification's name", }, + &cli.StringFlag{ + Name: "spec-version", + Usage: "use `VERSION` as specification's version", + Value: "0.0.0", + }, &cli.StringFlag{ Name: "spec-data", Usage: "use `DATA` as specification's data, '-' to read from STDIN", @@ -44,7 +49,12 @@ func UpdateCommand() *cli.Command { return errors.WithStack(err) } - specName, err := assertSpecName(ctx) + specDefName, err := assertSpecDefName(ctx) + if err != nil { + return errors.WithStack(err) + } + + specDefVersion, err := assertSpecDefVersion(ctx) if err != nil { return errors.WithStack(err) } @@ -71,7 +81,7 @@ func UpdateCommand() *cli.Command { var existingSpec spec.Spec for _, s := range specs { - if s.SpecName() != specName { + if s.SpecDefinitionName() != specDefName || s.SpecDefinitionVersion() != specDefVersion { continue } @@ -100,13 +110,10 @@ func UpdateCommand() *cli.Command { } rawSpec := &spec.RawSpec{ - Name: specName, - Revision: revision, - Data: specData, - } - - if err := spec.Validate(ctx.Context, rawSpec); err != nil { - return errors.WithStack(apierr.Wrap(err)) + DefinitionName: specDefName, + DefinitionVersion: specDefVersion, + Revision: revision, + Data: specData, } spec, err := client.UpdateAgentSpec(ctx.Context, agentID, rawSpec) @@ -114,9 +121,7 @@ func UpdateCommand() *cli.Command { return errors.WithStack(apierr.Wrap(err)) } - hints := format.Hints{ - OutputMode: baseFlags.OutputMode, - } + hints := specHints(baseFlags.OutputMode) if err := format.Write(baseFlags.Format, os.Stdout, hints, spec); err != nil { return errors.WithStack(err) @@ -127,14 +132,24 @@ func UpdateCommand() *cli.Command { } } -func assertSpecName(ctx *cli.Context) (spec.Name, error) { - specName := ctx.String("spec-name") +func assertSpecDefName(ctx *cli.Context) (string, error) { + specDefName := ctx.String("spec-name") - if specName == "" { + if specDefName == "" { return "", errors.New("flag 'spec-name' is required") } - return spec.Name(specName), nil + return specDefName, nil +} + +func assertSpecDefVersion(ctx *cli.Context) (string, error) { + specDefVersion := ctx.String("spec-version") + + if specDefVersion == "" { + return "", errors.New("flag 'spec-name' is required") + } + + return specDefVersion, nil } func assertSpecData(ctx *cli.Context) (map[string]any, error) { diff --git a/internal/datastore/agent_repository.go b/internal/datastore/agent_repository.go index 311d9cc..59c943e 100644 --- a/internal/datastore/agent_repository.go +++ b/internal/datastore/agent_repository.go @@ -18,9 +18,9 @@ type AgentRepository interface { Query(ctx context.Context, opts ...AgentQueryOptionFunc) ([]*Agent, int, error) Delete(ctx context.Context, id AgentID) error - UpdateSpec(ctx context.Context, id AgentID, name string, revision int, data map[string]any) (*Spec, error) + UpdateSpec(ctx context.Context, id AgentID, name string, version string, revision int, data map[string]any) (*Spec, error) GetSpecs(ctx context.Context, id AgentID) ([]*Spec, error) - DeleteSpec(ctx context.Context, id AgentID, name string) error + DeleteSpec(ctx context.Context, id AgentID, name string, version string) error } type AgentQueryOptionFunc func(*AgentQueryOptions) diff --git a/internal/datastore/memory/spec_definition_repository.go b/internal/datastore/memory/spec_definition_repository.go new file mode 100644 index 0000000..be6c241 --- /dev/null +++ b/internal/datastore/memory/spec_definition_repository.go @@ -0,0 +1,166 @@ +package memory + +import ( + "context" + "slices" + "sync" + "time" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" +) + +type specDefRecord struct { + Schema []byte + CreatedAt time.Time + UpdatedAt time.Time +} + +type SpecDefinitionRepository struct { + definitions map[string]map[string]specDefRecord + mutex sync.RWMutex +} + +// Delete implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Delete(ctx context.Context, name string, version string) error { + r.mutex.Lock() + defer r.mutex.Unlock() + + versions, exists := r.definitions[name] + if !exists { + return nil + } + + delete(versions, version) + + r.definitions[name] = versions + + return nil +} + +// Get implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Get(ctx context.Context, name string, version string) (*datastore.SpecDefinition, error) { + r.mutex.RLock() + defer r.mutex.RUnlock() + + versions, exists := r.definitions[name] + if !exists { + return nil, errors.WithStack(datastore.ErrNotFound) + } + + rec, exists := versions[version] + if !exists { + return nil, errors.WithStack(datastore.ErrNotFound) + } + + specDef := datastore.SpecDefinition{ + SpecDefinitionHeader: datastore.SpecDefinitionHeader{ + Name: name, + Version: version, + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + }, + Schema: rec.Schema[:], + } + + return &specDef, nil +} + +// Query implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Query(ctx context.Context, opts ...datastore.SpecDefinitionQueryOptionFunc) ([]datastore.SpecDefinitionHeader, int, error) { + options := &datastore.SpecDefinitionQueryOptions{} + for _, fn := range opts { + fn(options) + } + + r.mutex.RLock() + defer r.mutex.RUnlock() + + specDefs := make([]datastore.SpecDefinitionHeader, 0) + count := 0 + + for name, versions := range r.definitions { + for version, rec := range versions { + count++ + + matches := true + + if options.Names != nil && !slices.Contains(options.Names, name) { + matches = false + } + + if options.Versions != nil && !slices.Contains(options.Versions, version) { + matches = false + } + + if options.Offset != nil && count < *options.Offset { + matches = false + } + + if options.Limit != nil && len(specDefs) >= *options.Limit { + matches = false + } + + if !matches { + continue + } + + specDefs = append(specDefs, datastore.SpecDefinitionHeader{ + Name: name, + Version: version, + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + }) + } + } + + return specDefs, count, nil +} + +// Upsert implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Upsert(ctx context.Context, name string, version string, schema []byte) (*datastore.SpecDefinition, error) { + r.mutex.Lock() + defer r.mutex.Unlock() + + versions, exists := r.definitions[name] + if !exists { + versions = make(map[string]specDefRecord) + } + + now := time.Now().UTC() + + rec, exists := versions[version] + if !exists { + rec = specDefRecord{ + CreatedAt: now, + UpdatedAt: now, + Schema: schema[:], + } + } else { + rec.UpdatedAt = now + rec.Schema = schema + } + + versions[version] = rec + r.definitions[name] = versions + + specDef := datastore.SpecDefinition{ + SpecDefinitionHeader: datastore.SpecDefinitionHeader{ + Name: name, + Version: version, + CreatedAt: rec.CreatedAt, + UpdatedAt: rec.UpdatedAt, + }, + Schema: rec.Schema[:], + } + + return &specDef, nil +} + +func NewSpecDefinitionRepository() *SpecDefinitionRepository { + return &SpecDefinitionRepository{ + definitions: make(map[string]map[string]specDefRecord), + } +} + +var _ datastore.SpecDefinitionRepository = &SpecDefinitionRepository{} diff --git a/internal/datastore/memory/spec_definition_repository_test.go b/internal/datastore/memory/spec_definition_repository_test.go new file mode 100644 index 0000000..df6c008 --- /dev/null +++ b/internal/datastore/memory/spec_definition_repository_test.go @@ -0,0 +1,14 @@ +package memory + +import ( + "testing" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore/testsuite" + "gitlab.com/wpetit/goweb/logger" +) + +func TestMemorySpecDefinitionRepository(t *testing.T) { + logger.SetLevel(logger.LevelDebug) + repo := NewSpecDefinitionRepository() + testsuite.TestSpecDefinitionRepository(t, repo) +} diff --git a/internal/datastore/spec.go b/internal/datastore/spec.go index be417c7..c04c426 100644 --- a/internal/datastore/spec.go +++ b/internal/datastore/spec.go @@ -2,25 +2,28 @@ package datastore import ( "time" - - "forge.cadoles.com/Cadoles/emissary/internal/spec" ) type SpecID int64 type Spec struct { - ID SpecID `json:"id"` - Name string `json:"name"` - Data map[string]any `json:"data"` - Revision int `json:"revision"` - CreatedAt time.Time `json:"createdAt"` - UpdatedAt time.Time `json:"updatedAt"` - TenantID TenantID `json:"tenantId"` - AgentID AgentID `json:"agentId"` + ID SpecID `json:"id"` + DefinitionName string `json:"name"` + DefinitionVersion string `json:"version"` + Data map[string]any `json:"data"` + Revision int `json:"revision"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + TenantID TenantID `json:"tenantId"` + AgentID AgentID `json:"agentId"` } -func (s *Spec) SpecName() spec.Name { - return spec.Name(s.Name) +func (s *Spec) SpecDefinitionName() string { + return s.DefinitionName +} + +func (s *Spec) SpecDefinitionVersion() string { + return s.DefinitionVersion } func (s *Spec) SpecRevision() int { diff --git a/internal/datastore/spec_definition.go b/internal/datastore/spec_definition.go new file mode 100644 index 0000000..376a83b --- /dev/null +++ b/internal/datastore/spec_definition.go @@ -0,0 +1,18 @@ +package datastore + +import ( + "time" +) + +type SpecDefinitionHeader struct { + Name string `json:"name"` + Version string `json:"version"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type SpecDefinition struct { + SpecDefinitionHeader + + Schema []byte `json:"schema"` +} diff --git a/internal/datastore/spec_definition_repository.go b/internal/datastore/spec_definition_repository.go new file mode 100644 index 0000000..4bc3db0 --- /dev/null +++ b/internal/datastore/spec_definition_repository.go @@ -0,0 +1,46 @@ +package datastore + +import ( + "context" +) + +type SpecDefinitionRepository interface { + Upsert(ctx context.Context, name string, version string, schema []byte) (*SpecDefinition, error) + Get(ctx context.Context, name string, version string) (*SpecDefinition, error) + Delete(ctx context.Context, name string, version string) error + + Query(ctx context.Context, opts ...SpecDefinitionQueryOptionFunc) ([]SpecDefinitionHeader, int, error) +} + +type SpecDefinitionQueryOptionFunc func(*SpecDefinitionQueryOptions) + +type SpecDefinitionQueryOptions struct { + Limit *int + Offset *int + Names []string + Versions []string +} + +func WithSpecDefinitionQueryLimit(limit int) SpecDefinitionQueryOptionFunc { + return func(opts *SpecDefinitionQueryOptions) { + opts.Limit = &limit + } +} + +func WithSpecDefinitionQueryOffset(offset int) SpecDefinitionQueryOptionFunc { + return func(opts *SpecDefinitionQueryOptions) { + opts.Offset = &offset + } +} + +func WithSpecDefinitionQueryNames(names ...string) SpecDefinitionQueryOptionFunc { + return func(opts *SpecDefinitionQueryOptions) { + opts.Names = names + } +} + +func WithSpecDefinitionQueryVersions(versions ...string) SpecDefinitionQueryOptionFunc { + return func(opts *SpecDefinitionQueryOptions) { + opts.Versions = versions + } +} diff --git a/internal/datastore/sqlite/agent_repository.go b/internal/datastore/sqlite/agent_repository.go index cc8ba50..73e9407 100644 --- a/internal/datastore/sqlite/agent_repository.go +++ b/internal/datastore/sqlite/agent_repository.go @@ -128,7 +128,7 @@ func (r *AgentRepository) Detach(ctx context.Context, agentID datastore.AgentID) } // DeleteSpec implements datastore.AgentRepository. -func (r *AgentRepository) DeleteSpec(ctx context.Context, agentID datastore.AgentID, name string) error { +func (r *AgentRepository) DeleteSpec(ctx context.Context, agentID datastore.AgentID, name string, version string) error { err := r.withTxRetry(ctx, func(tx *sql.Tx) error { exists, err := r.agentExists(ctx, tx, agentID) if err != nil { @@ -139,9 +139,9 @@ func (r *AgentRepository) DeleteSpec(ctx context.Context, agentID datastore.Agen return errors.WithStack(datastore.ErrNotFound) } - query := `DELETE FROM specs WHERE agent_id = $1 AND name = $2` + query := `DELETE FROM specs WHERE agent_id = $1 AND name = $2 AND version = $3` - if _, err = tx.ExecContext(ctx, query, agentID, name); err != nil { + if _, err = tx.ExecContext(ctx, query, agentID, name, version); err != nil { return errors.WithStack(err) } @@ -169,7 +169,7 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI } query := ` - SELECT id, name, revision, data, created_at, updated_at + SELECT id, name, version, revision, data, created_at, updated_at, agent_id, tenant_id FROM specs WHERE agent_id = $1 ` @@ -191,10 +191,14 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI data := JSONMap{} - if err := rows.Scan(&spec.ID, &spec.Name, &spec.Revision, &data, &spec.CreatedAt, &spec.UpdatedAt); err != nil { + var tenantID sql.NullString + if err := rows.Scan(&spec.ID, &spec.DefinitionName, &spec.DefinitionVersion, &spec.Revision, &data, &spec.CreatedAt, &spec.UpdatedAt, &spec.AgentID, &tenantID); err != nil { return errors.WithStack(err) } + if tenantID.Valid { + spec.TenantID = datastore.TenantID(tenantID.String) + } spec.Data = data specs = append(specs, spec) @@ -214,7 +218,7 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI } // UpdateSpec implements datastore.AgentRepository. -func (r *AgentRepository) UpdateSpec(ctx context.Context, agentID datastore.AgentID, name string, revision int, data map[string]any) (*datastore.Spec, error) { +func (r *AgentRepository) UpdateSpec(ctx context.Context, agentID datastore.AgentID, name string, version string, revision int, data map[string]any) (*datastore.Spec, error) { spec := &datastore.Spec{} err := r.withTxRetry(ctx, func(tx *sql.Tx) error { @@ -230,23 +234,24 @@ func (r *AgentRepository) UpdateSpec(ctx context.Context, agentID datastore.Agen now := time.Now().UTC() query := ` - INSERT INTO specs (agent_id, name, revision, data, created_at, updated_at, tenant_id) - VALUES($1, $2, $3, $4, $5, $5, ( SELECT tenant_id FROM agents WHERE id = $1 )) - ON CONFLICT (agent_id, name) DO UPDATE SET - data = $4, updated_at = $5, revision = specs.revision + 1 - WHERE revision = $3 - RETURNING "id", "name", "revision", "data", "created_at", "updated_at" + INSERT INTO specs (agent_id, name, version, revision, data, created_at, updated_at, tenant_id) + VALUES($1, $2, $3, $4, $5, $6, $6, ( SELECT tenant_id FROM agents WHERE id = $1 )) + ON CONFLICT (agent_id, name, version) DO UPDATE SET + data = $5, updated_at = $6, revision = specs.revision + 1, tenant_id = ( SELECT tenant_id FROM agents WHERE id = $1 ) + WHERE revision = $4 + RETURNING "id", "name", "version", "revision", "data", "created_at", "updated_at", "tenant_id", "agent_id" ` - args := []any{agentID, name, revision, JSONMap(data), now} + args := []any{agentID, name, version, revision, JSONMap(data), now} logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args)) row := tx.QueryRowContext(ctx, query, args...) data := JSONMap{} + var tenantID sql.NullString - err = row.Scan(&spec.ID, &spec.Name, &spec.Revision, &data, &spec.CreatedAt, &spec.UpdatedAt) + err = row.Scan(&spec.ID, &spec.DefinitionName, &spec.DefinitionVersion, &spec.Revision, &data, &spec.CreatedAt, &spec.UpdatedAt, &tenantID, &spec.AgentID) if err != nil { if errors.Is(err, sql.ErrNoRows) { return errors.WithStack(datastore.ErrUnexpectedRevision) @@ -255,6 +260,10 @@ func (r *AgentRepository) UpdateSpec(ctx context.Context, agentID datastore.Agen return errors.WithStack(err) } + if tenantID.Valid { + spec.TenantID = datastore.TenantID(tenantID.String) + } + spec.Data = data return nil @@ -301,6 +310,10 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer } if options.TenantIDs != nil && len(options.TenantIDs) > 0 { + if filters != "" { + filters += " AND " + } + filter, newArgs, newParamIndex := inFilter("tenant_id", paramIndex, options.TenantIDs) filters += filter paramIndex = newParamIndex diff --git a/internal/datastore/sqlite/json.go b/internal/datastore/sqlite/json.go index 2bfc5a3..e73e294 100644 --- a/internal/datastore/sqlite/json.go +++ b/internal/datastore/sqlite/json.go @@ -7,6 +7,42 @@ import ( "github.com/pkg/errors" ) +type JSON struct { + value any +} + +func (j JSON) Scan(value interface{}) error { + if value == nil { + return nil + } + + var data []byte + + switch typ := value.(type) { + case []byte: + data = typ + case string: + data = []byte(typ) + default: + return errors.Errorf("unexpected type '%T'", value) + } + + if err := json.Unmarshal(data, &j.value); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func (j JSON) Value() (driver.Value, error) { + data, err := json.Marshal(j.value) + if err != nil { + return nil, errors.WithStack(err) + } + + return data, nil +} + type JSONMap map[string]any func (j *JSONMap) Scan(value interface{}) error { diff --git a/internal/datastore/sqlite/spec_definition_repository.go b/internal/datastore/sqlite/spec_definition_repository.go new file mode 100644 index 0000000..ffb24ae --- /dev/null +++ b/internal/datastore/sqlite/spec_definition_repository.go @@ -0,0 +1,219 @@ +package sqlite + +import ( + "context" + "database/sql" + "time" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" +) + +type SpecDefinitionRepository struct { + repository +} + +// Delete implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Delete(ctx context.Context, name string, version string) error { + err := r.withTxRetry(ctx, func(tx *sql.Tx) error { + if exists, err := r.specDefinitionExists(ctx, tx, name, version); !exists { + return errors.WithStack(err) + } + + query := `DELETE FROM spec_definitions WHERE name = $1 AND version = $2` + _, err := tx.ExecContext(ctx, query, name, version) + if err != nil { + return errors.WithStack(err) + } + + return nil + }) + if err != nil { + return errors.WithStack(err) + } + + return nil +} + +// Get implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Get(ctx context.Context, name string, version string) (*datastore.SpecDefinition, error) { + var specDef datastore.SpecDefinition + + err := r.withTxRetry(ctx, func(tx *sql.Tx) error { + query := ` + SELECT "name", "version", "schema", "created_at", "updated_at" + FROM spec_definitions + WHERE name = $1 AND version = $2 + ` + + row := tx.QueryRowContext(ctx, query, name, version) + + if err := row.Scan(&specDef.Name, &specDef.Version, &specDef.Schema, &specDef.CreatedAt, &specDef.UpdatedAt); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return errors.WithStack(datastore.ErrNotFound) + } + + return errors.WithStack(err) + } + + return nil + }) + if err != nil { + return nil, errors.WithStack(err) + } + + return &specDef, nil +} + +// Query implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Query(ctx context.Context, opts ...datastore.SpecDefinitionQueryOptionFunc) ([]datastore.SpecDefinitionHeader, int, error) { + options := &datastore.SpecDefinitionQueryOptions{} + for _, fn := range opts { + fn(options) + } + + specDefs := make([]datastore.SpecDefinitionHeader, 0) + count := 0 + + err := r.withTxRetry(ctx, func(tx *sql.Tx) error { + query := `SELECT name, version, created_at, updated_at FROM spec_definitions` + + limit := 10 + if options.Limit != nil { + limit = *options.Limit + } + + offset := 0 + if options.Offset != nil { + offset = *options.Offset + } + + filters := "" + paramIndex := 3 + args := []any{offset, limit} + + if options.Names != nil && len(options.Names) > 0 { + filter, newArgs, newParamIndex := inFilter("name", paramIndex, options.Names) + filters += filter + paramIndex = newParamIndex + args = append(args, newArgs...) + } + + if options.Versions != nil && len(options.Versions) > 0 { + if filters != "" { + filters += " AND " + } + + filter, newArgs, _ := inFilter("version", paramIndex, options.Versions) + filters += filter + args = append(args, newArgs...) + } + + if filters != "" { + filters = ` WHERE ` + filters + } + + query += filters + ` LIMIT $2 OFFSET $1` + + logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args)) + + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return errors.WithStack(err) + } + + defer func() { + if err := rows.Close(); err != nil { + err = errors.WithStack(err) + logger.Error(ctx, "could not close rows", logger.CapturedE(err)) + } + }() + + for rows.Next() { + sdh := datastore.SpecDefinitionHeader{} + + if err := rows.Scan(&sdh.Name, &sdh.Version, &sdh.CreatedAt, &sdh.UpdatedAt); err != nil { + return errors.WithStack(err) + } + + specDefs = append(specDefs, sdh) + } + + if err := rows.Err(); err != nil { + return errors.WithStack(err) + } + + row := tx.QueryRowContext(ctx, `SELECT count(*) FROM spec_definitions `+filters, args...) + if err := row.Scan(&count); err != nil { + return errors.WithStack(err) + } + + return nil + }) + if err != nil { + return nil, 0, errors.WithStack(err) + } + + return specDefs, count, nil +} + +// Upsert implements datastore.SpecDefinitionRepository. +func (r *SpecDefinitionRepository) Upsert(ctx context.Context, name string, version string, schema []byte) (*datastore.SpecDefinition, error) { + var specDef datastore.SpecDefinition + + err := r.withTxRetry(ctx, func(tx *sql.Tx) error { + now := time.Now().UTC() + + query := ` + INSERT INTO spec_definitions (name, version, schema, created_at, updated_at) + VALUES($1, $2, $3, $4, $4) + ON CONFLICT(name, version) DO UPDATE SET schema = $3, updated_at = $4 + RETURNING "name", "version", "schema", "created_at", "updated_at" + ` + + row := tx.QueryRowContext( + ctx, query, + name, version, schema, now, now, + ) + + if err := row.Scan(&specDef.Name, &specDef.Version, &specDef.Schema, &specDef.CreatedAt, &specDef.UpdatedAt); err != nil { + return errors.WithStack(err) + } + + return nil + }) + if err != nil { + return nil, errors.WithStack(err) + } + + return &specDef, nil +} + +func (r *SpecDefinitionRepository) specDefinitionExists(ctx context.Context, tx *sql.Tx, name string, version string) (bool, error) { + row := tx.QueryRowContext(ctx, `SELECT count(id) FROM spec_definitions WHERE name = $1 AND version = $2`, name, version) + + var count int + + if err := row.Scan(&count); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, errors.WithStack(datastore.ErrNotFound) + } + + return false, errors.WithStack(err) + } + + if count == 0 { + return false, errors.WithStack(datastore.ErrNotFound) + } + + return true, nil +} + +func NewSpecDefinitionRepository(db *sql.DB, sqliteBusyRetryMaxAttempts int) *SpecDefinitionRepository { + return &SpecDefinitionRepository{ + repository: repository{db, sqliteBusyRetryMaxAttempts}, + } +} + +var _ datastore.SpecDefinitionRepository = &SpecDefinitionRepository{} diff --git a/internal/datastore/sqlite/spec_definition_repository_test.go b/internal/datastore/sqlite/spec_definition_repository_test.go new file mode 100644 index 0000000..d0f1803 --- /dev/null +++ b/internal/datastore/sqlite/spec_definition_repository_test.go @@ -0,0 +1,46 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "os" + "testing" + "time" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore/testsuite" + "forge.cadoles.com/Cadoles/emissary/internal/migrate" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" + + _ "modernc.org/sqlite" +) + +func TestSQLiteSpecDefinitionRepository(t *testing.T) { + logger.SetLevel(logger.LevelDebug) + + file := "testdata/spec_definition_repository_test.sqlite" + + if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("%+v", errors.WithStack(err)) + } + + dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds()) + + migr, err := migrate.New("../../../migrations", "sqlite", "sqlite://"+dsn) + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + if err := migr.Up(); err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("%+v", errors.WithStack(err)) + } + + repo := NewSpecDefinitionRepository(db, 5) + + testsuite.TestSpecDefinitionRepository(t, repo) +} diff --git a/internal/datastore/sqlite/tenant_repository_test.go b/internal/datastore/sqlite/tenant_repository_test.go index ce4a9f9..05efd90 100644 --- a/internal/datastore/sqlite/tenant_repository_test.go +++ b/internal/datastore/sqlite/tenant_repository_test.go @@ -15,7 +15,7 @@ import ( _ "modernc.org/sqlite" ) -func TestSQLiteTeantRepository(t *testing.T) { +func TestSQLiteTenantRepository(t *testing.T) { logger.SetLevel(logger.LevelDebug) file := "testdata/tenant_repository_test.sqlite" diff --git a/internal/datastore/testsuite/agent_repository_cases.go b/internal/datastore/testsuite/agent_repository_cases.go index d682062..0364542 100644 --- a/internal/datastore/testsuite/agent_repository_cases.go +++ b/internal/datastore/testsuite/agent_repository_cases.go @@ -50,7 +50,7 @@ var agentRepositoryTestCases = []agentRepositoryTestCase{ var unexistantAgentID datastore.AgentID = 9999 var specData map[string]any - agent, err := repo.UpdateSpec(ctx, unexistantAgentID, string(spec.Name), 0, specData) + agent, err := repo.UpdateSpec(ctx, unexistantAgentID, spec.Name, spec.Version, 0, specData) if err == nil { return errors.New("error should not be nil") } @@ -71,7 +71,7 @@ var agentRepositoryTestCases = []agentRepositoryTestCase{ Run: func(ctx context.Context, repo datastore.AgentRepository) error { var unexistantAgentID datastore.AgentID = 9999 - err := repo.DeleteSpec(ctx, unexistantAgentID, string(spec.Name)) + err := repo.DeleteSpec(ctx, unexistantAgentID, spec.Name, spec.Version) if err == nil { return errors.New("error should not be nil") } diff --git a/internal/datastore/testsuite/spec_definition_repository.go b/internal/datastore/testsuite/spec_definition_repository.go new file mode 100644 index 0000000..cb49bd6 --- /dev/null +++ b/internal/datastore/testsuite/spec_definition_repository.go @@ -0,0 +1,14 @@ +package testsuite + +import ( + "testing" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" +) + +func TestSpecDefinitionRepository(t *testing.T, repo datastore.SpecDefinitionRepository) { + t.Run("Cases", func(t *testing.T) { + t.Parallel() + runSpecDefinitionRepositoryTests(t, repo) + }) +} diff --git a/internal/datastore/testsuite/spec_definition_repository_cases.go b/internal/datastore/testsuite/spec_definition_repository_cases.go new file mode 100644 index 0000000..d040449 --- /dev/null +++ b/internal/datastore/testsuite/spec_definition_repository_cases.go @@ -0,0 +1,76 @@ +package testsuite + +import ( + "context" + "reflect" + "testing" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" +) + +type specDefinitionRepositoryTestCase struct { + Name string + Skip bool + Run func(ctx context.Context, repo datastore.SpecDefinitionRepository) error +} + +var specDefinitionRepositoryTestCases = []specDefinitionRepositoryTestCase{ + { + Name: "Create a spec definition", + Run: func(ctx context.Context, repo datastore.SpecDefinitionRepository) error { + schema := []byte("{}") + name := "net.example.foo" + version := "0.0.0" + + specDef, err := repo.Upsert(ctx, name, version, schema) + if err != nil { + return errors.WithStack(err) + } + + if specDef.CreatedAt.IsZero() { + return errors.Errorf("specDef.CreatedAt should not be zero time") + } + + if specDef.UpdatedAt.IsZero() { + return errors.Errorf("specDef.UpdatedAt should not be zero time") + } + + if e, g := name, specDef.Name; e != g { + return errors.Errorf("specDef.Name: expected '%v', got '%v'", e, g) + } + + if e, g := version, specDef.Version; e != g { + return errors.Errorf("specDef.Name: expected '%v', got '%v'", e, g) + } + + if e, g := schema, specDef.Schema; !reflect.DeepEqual(e, g) { + return errors.Errorf("specDef.Schema: expected '%v', got '%v'", e, g) + } + + return nil + }, + }, +} + +func runSpecDefinitionRepositoryTests(t *testing.T, repo datastore.SpecDefinitionRepository) { + for _, tc := range specDefinitionRepositoryTestCases { + func(tc specDefinitionRepositoryTestCase) { + t.Run(tc.Name, func(t *testing.T) { + t.Parallel() + + if tc.Skip { + t.SkipNow() + + return + } + + ctx := context.Background() + + if err := tc.Run(ctx, repo); err != nil { + t.Errorf("%+v", errors.WithStack(err)) + } + }) + }(tc) + } +} diff --git a/internal/server/api/get_specs.go b/internal/server/api/delete_agent_spec.go similarity index 51% rename from internal/server/api/get_specs.go rename to internal/server/api/delete_agent_spec.go index cc81625..7e35788 100644 --- a/internal/server/api/get_specs.go +++ b/internal/server/api/delete_agent_spec.go @@ -9,42 +9,12 @@ import ( "gitlab.com/wpetit/goweb/logger" ) -func (m *Mount) getAgentSpecs(w http.ResponseWriter, r *http.Request) { - agentID, ok := getAgentID(w, r) - if !ok { - return - } - - ctx := r.Context() - - specs, err := m.agentRepo.GetSpecs(ctx, agentID) - if err != nil { - if errors.Is(err, datastore.ErrNotFound) { - api.ErrorResponse(w, http.StatusNotFound, ErrCodeNotFound, nil) - - return - } - - err = errors.WithStack(err) - logger.Error(ctx, "could not list specs", logger.CapturedE(err)) - - api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil) - - return - } - - api.DataResponse(w, http.StatusOK, struct { - Specs []*datastore.Spec `json:"specs"` - }{ - Specs: specs, - }) -} - type deleteSpecRequest struct { - Name string `json:"name"` + Name string `json:"name" validate:"required"` + Version string `json:"version" validate:"required"` } -func (m *Mount) deleteSpec(w http.ResponseWriter, r *http.Request) { +func (m *Mount) deleteAgentSpec(w http.ResponseWriter, r *http.Request) { agentID, ok := getAgentID(w, r) if !ok { return @@ -61,6 +31,7 @@ func (m *Mount) deleteSpec(w http.ResponseWriter, r *http.Request) { ctx, agentID, deleteSpecReq.Name, + deleteSpecReq.Version, ) if err != nil { if errors.Is(err, datastore.ErrNotFound) { @@ -78,8 +49,10 @@ func (m *Mount) deleteSpec(w http.ResponseWriter, r *http.Request) { } api.DataResponse(w, http.StatusOK, struct { - Name string `json:"name"` + Name string `json:"name"` + Version string `json:"version"` }{ - Name: deleteSpecReq.Name, + Name: deleteSpecReq.Name, + Version: deleteSpecReq.Version, }) } diff --git a/internal/server/api/get_agent_specs.go b/internal/server/api/get_agent_specs.go new file mode 100644 index 0000000..fce7a56 --- /dev/null +++ b/internal/server/api/get_agent_specs.go @@ -0,0 +1,41 @@ +package api + +import ( + "net/http" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/api" + "gitlab.com/wpetit/goweb/logger" +) + +func (m *Mount) getAgentSpecs(w http.ResponseWriter, r *http.Request) { + agentID, ok := getAgentID(w, r) + if !ok { + return + } + + ctx := r.Context() + + specs, err := m.agentRepo.GetSpecs(ctx, agentID) + if err != nil { + if errors.Is(err, datastore.ErrNotFound) { + api.ErrorResponse(w, http.StatusNotFound, ErrCodeNotFound, nil) + + return + } + + err = errors.WithStack(err) + logger.Error(ctx, "could not list specs", logger.CapturedE(err)) + + api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil) + + return + } + + api.DataResponse(w, http.StatusOK, struct { + Specs []*datastore.Spec `json:"specs"` + }{ + Specs: specs, + }) +} diff --git a/internal/server/api/get_spec_definition.go b/internal/server/api/get_spec_definition.go new file mode 100644 index 0000000..74802de --- /dev/null +++ b/internal/server/api/get_spec_definition.go @@ -0,0 +1,44 @@ +package api + +import ( + "net/http" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/api" + "gitlab.com/wpetit/goweb/logger" +) + +func (m *Mount) getSpecDefinition(w http.ResponseWriter, r *http.Request) { + specDefName, specDefVersion, ok := getSpecDefinitionNameAndVersion(w, r) + if !ok { + return + } + + ctx := r.Context() + + specDef, err := m.specDefRepo.Get( + ctx, + specDefName, + specDefVersion, + ) + if err != nil { + if errors.Is(err, datastore.ErrNotFound) { + api.ErrorResponse(w, http.StatusNotFound, ErrCodeNotFound, nil) + + return + } + + err = errors.WithStack(err) + logger.Error(ctx, "could not get agent", logger.CapturedE(err)) + api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil) + + return + } + + api.DataResponse(w, http.StatusOK, struct { + SpecDefinition *datastore.SpecDefinition `json:"specDefinition"` + }{ + SpecDefinition: specDef, + }) +} diff --git a/internal/server/api/helper.go b/internal/server/api/helper.go index 7376e55..9fda6d1 100644 --- a/internal/server/api/helper.go +++ b/internal/server/api/helper.go @@ -35,20 +35,11 @@ func getAgentID(w http.ResponseWriter, r *http.Request) (datastore.AgentID, bool return datastore.AgentID(agentID), true } -func getSpecID(w http.ResponseWriter, r *http.Request) (datastore.SpecID, bool) { - rawSpecID := chi.URLParam(r, "specID") +func getSpecDefinitionNameAndVersion(w http.ResponseWriter, r *http.Request) (string, string, bool) { + specDefName := chi.URLParam(r, "specDefName") + specDefVersion := chi.URLParam(r, "specDefVersion") - specID, err := strconv.ParseInt(rawSpecID, 10, 64) - if err != nil { - err = errors.WithStack(err) - logger.Error(r.Context(), "could not parse spec id", logger.CapturedE(err)) - - api.ErrorResponse(w, http.StatusBadRequest, api.ErrCodeMalformedRequest, nil) - - return 0, false - } - - return datastore.SpecID(specID), true + return specDefName, specDefVersion, true } func getTenantID(w http.ResponseWriter, r *http.Request) (datastore.TenantID, bool) { diff --git a/internal/server/api/mount.go b/internal/server/api/mount.go index ff1aa14..178fcf7 100644 --- a/internal/server/api/mount.go +++ b/internal/server/api/mount.go @@ -12,6 +12,7 @@ import ( type Mount struct { agentRepo datastore.AgentRepository tenantRepo datastore.TenantRepository + specDefRepo datastore.SpecDefinitionRepository authenticators []auth.Authenticator } @@ -35,8 +36,8 @@ func (m *Mount) Mount(r chi.Router) { r.With(assertUserWithWriteAccess).Delete("/{agentID}", m.deleteAgent) r.With(assertAgentOrUserWithReadAccess).Get("/{agentID}/specs", m.getAgentSpecs) - r.With(assertUserWithWriteAccess).Post("/{agentID}/specs", m.updateSpec) - r.With(assertUserWithWriteAccess).Delete("/{agentID}/specs", m.deleteSpec) + r.With(assertUserWithWriteAccess).Post("/{agentID}/specs", m.updateAgentSpec) + r.With(assertUserWithWriteAccess).Delete("/{agentID}/specs", m.deleteAgentSpec) }) r.Route("/tenants", func(r chi.Router) { @@ -46,6 +47,11 @@ func (m *Mount) Mount(r chi.Router) { r.With(assertAdminOrTenantWriteAccess).Put("/{tenantID}", m.updateTenant) r.With(assertAdminAccess).Delete("/{tenantID}", m.deleteTenant) }) + + r.Route("/specs", func(r chi.Router) { + r.With(assertQueryAccess).Get("/", m.querySpecDefinitions) + r.With(assertQueryAccess).Get("/{specDefName}/{specDefVersion}", m.getSpecDefinition) + }) }) } @@ -53,6 +59,6 @@ func (m *Mount) notFound(w http.ResponseWriter, r *http.Request) { api.ErrorResponse(w, http.StatusNotFound, ErrCodeNotFound, nil) } -func NewMount(agentRepo datastore.AgentRepository, tenantRepo datastore.TenantRepository, authenticators ...auth.Authenticator) *Mount { - return &Mount{agentRepo, tenantRepo, authenticators} +func NewMount(agentRepo datastore.AgentRepository, tenantRepo datastore.TenantRepository, specDefRepo datastore.SpecDefinitionRepository, authenticators ...auth.Authenticator) *Mount { + return &Mount{agentRepo, tenantRepo, specDefRepo, authenticators} } diff --git a/internal/server/api/query_spec_definitions.go b/internal/server/api/query_spec_definitions.go new file mode 100644 index 0000000..b3ea1ef --- /dev/null +++ b/internal/server/api/query_spec_definitions.go @@ -0,0 +1,67 @@ +package api + +import ( + "net/http" + + "forge.cadoles.com/Cadoles/emissary/internal/datastore" + "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/api" + "gitlab.com/wpetit/goweb/logger" +) + +func (m *Mount) querySpecDefinitions(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + limit, ok := getIntQueryParam(w, r, "limit", 10) + if !ok { + return + } + + offset, ok := getIntQueryParam(w, r, "offset", 0) + if !ok { + return + } + + options := []datastore.SpecDefinitionQueryOptionFunc{ + datastore.WithSpecDefinitionQueryLimit(int(limit)), + datastore.WithSpecDefinitionQueryOffset(int(offset)), + } + + names, ok := getStringSliceValues(w, r, "names", nil) + if !ok { + return + } + + if len(names) > 0 { + options = append(options, datastore.WithSpecDefinitionQueryNames(names...)) + } + + versions, ok := getStringSliceValues(w, r, "versions", nil) + if !ok { + return + } + + if len(names) > 0 { + options = append(options, datastore.WithSpecDefinitionQueryVersions(versions...)) + } + + specDefinitions, total, err := m.specDefRepo.Query( + ctx, + options..., + ) + if err != nil { + err = errors.WithStack(err) + logger.Error(ctx, "could not list spec definitions", logger.CapturedE(err)) + api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil) + + return + } + + api.DataResponse(w, http.StatusOK, struct { + SpecDefinitions []datastore.SpecDefinitionHeader `json:"specDefinitions"` + Total int `json:"total"` + }{ + SpecDefinitions: specDefinitions, + Total: total, + }) +} diff --git a/internal/server/api/update_spec.go b/internal/server/api/update_agent_spec.go similarity index 77% rename from internal/server/api/update_spec.go rename to internal/server/api/update_agent_spec.go index 9a588e5..9c9043c 100644 --- a/internal/server/api/update_spec.go +++ b/internal/server/api/update_agent_spec.go @@ -14,11 +14,11 @@ const ( ErrCodeUnexpectedRevision api.ErrorCode = "unexpected-revision" ) -type updateSpecRequest struct { +type updateAgentSpecRequest struct { spec.RawSpec } -func (m *Mount) updateSpec(w http.ResponseWriter, r *http.Request) { +func (m *Mount) updateAgentSpec(w http.ResponseWriter, r *http.Request) { agentID, ok := getAgentID(w, r) if !ok { return @@ -26,12 +26,18 @@ func (m *Mount) updateSpec(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - updateSpecReq := &updateSpecRequest{} + updateSpecReq := &updateAgentSpecRequest{} if ok := api.Bind(w, r, updateSpecReq); !ok { return } - if err := spec.Validate(ctx, updateSpecReq); err != nil { + if updateSpecReq.DefinitionVersion == "" { + updateSpecReq.DefinitionVersion = spec.DefaultVersion + } + + validator := spec.NewValidator(m.specDefRepo) + + if err := validator.Validate(ctx, updateSpecReq); err != nil { data := struct { Message string `json:"message"` }{} @@ -53,7 +59,8 @@ func (m *Mount) updateSpec(w http.ResponseWriter, r *http.Request) { spec, err := m.agentRepo.UpdateSpec( ctx, datastore.AgentID(agentID), - string(updateSpecReq.SpecName()), + updateSpecReq.SpecDefinitionName(), + updateSpecReq.SpecDefinitionVersion(), updateSpecReq.SpecRevision(), updateSpecReq.SpecData(), ) diff --git a/internal/server/init.go b/internal/server/init.go index 72002c8..1083bb4 100644 --- a/internal/server/init.go +++ b/internal/server/init.go @@ -4,7 +4,9 @@ import ( "context" "forge.cadoles.com/Cadoles/emissary/internal/setup" + "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" + "gitlab.com/wpetit/goweb/logger" ) func (s *Server) initRepositories(ctx context.Context) error { @@ -18,8 +20,35 @@ func (s *Server) initRepositories(ctx context.Context) error { return errors.WithStack(err) } + specDefRepo, err := setup.NewSpecDefinitionRepository(ctx, s.conf.Database) + if err != nil { + return errors.WithStack(err) + } + s.agentRepo = agentRepo s.tenantRepo = tenantRepo + s.specDefRepo = specDefRepo + + return nil +} + +func (s *Server) initSpecDefinitions(ctx context.Context) error { + err := spec.Walk(func(name, version string, schema []byte) error { + logger.Debug( + ctx, "updating spec definition", + logger.F("name", name), + logger.F("version", version), + ) + if _, err := s.specDefRepo.Upsert(ctx, name, version, schema); err != nil { + return errors.WithStack(err) + } + + return nil + }) + + if err != nil { + return errors.WithStack(err) + } return nil } diff --git a/internal/server/server.go b/internal/server/server.go index e51468a..2347dd9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -28,9 +28,10 @@ import ( ) type Server struct { - conf config.ServerConfig - agentRepo datastore.AgentRepository - tenantRepo datastore.TenantRepository + conf config.ServerConfig + agentRepo datastore.AgentRepository + tenantRepo datastore.TenantRepository + specDefRepo datastore.SpecDefinitionRepository } func (s *Server) Start(ctx context.Context) (<-chan net.Addr, <-chan error) { @@ -57,6 +58,12 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e return } + if err := s.initSpecDefinitions(ctx); err != nil { + errs <- errors.WithStack(err) + + return + } + listener, err := net.Listen("tcp", fmt.Sprintf("%s:%d", s.conf.HTTP.Host, s.conf.HTTP.Port)) if err != nil { errs <- errors.WithStack(err) @@ -105,6 +112,7 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e apiMount := api.NewMount( s.agentRepo, s.tenantRepo, + s.specDefRepo, userAuth, agent.NewAuthenticator(s.agentRepo, agent.DefaultAcceptableSkew), ) diff --git a/internal/setup/repository.go b/internal/setup/repository.go index 3854db0..ff04906 100644 --- a/internal/setup/repository.go +++ b/internal/setup/repository.go @@ -104,3 +104,40 @@ func NewTenantRepository(ctx context.Context, conf config.DatabaseConfig) (datas return tenantRepository, nil } + +func NewSpecDefinitionRepository(ctx context.Context, conf config.DatabaseConfig) (datastore.SpecDefinitionRepository, error) { + driver := string(conf.Driver) + dsn := string(conf.DSN) + + var specDefRepository datastore.SpecDefinitionRepository + + logger.Debug(ctx, "initializing spec definition repository", logger.F("driver", driver), logger.F("dsn", dsn)) + + switch driver { + case config.DatabaseDriverPostgres: + // TODO + // pool, err := openPostgresPool(ctx, dsn) + // if err != nil { + // return nil, errors.WithStack(err) + // } + + // entryRepository = postgres.NewEntryRepository(pool) + case config.DatabaseDriverSQLite: + url, err := url.Parse(dsn) + if err != nil { + return nil, errors.WithStack(err) + } + + db, err := sql.Open(driver, url.Host+url.Path) + if err != nil { + return nil, errors.WithStack(err) + } + + specDefRepository = sqlite.NewSpecDefinitionRepository(db, 5) + + default: + return nil, errors.Errorf("unsupported database driver '%s'", driver) + } + + return specDefRepository, nil +} diff --git a/internal/spec/compare.go b/internal/spec/compare.go index 1b613c7..f949dc5 100644 --- a/internal/spec/compare.go +++ b/internal/spec/compare.go @@ -6,7 +6,11 @@ import ( ) func Equals(a Spec, b Spec) (bool, error) { - if a.SpecName() != b.SpecName() { + if a.SpecDefinitionName() != b.SpecDefinitionName() { + return false, nil + } + + if a.SpecDefinitionVersion() != b.SpecDefinitionVersion() { return false, nil } diff --git a/internal/spec/name.go b/internal/spec/name.go deleted file mode 100644 index fa7fd70..0000000 --- a/internal/spec/name.go +++ /dev/null @@ -1,3 +0,0 @@ -package spec - -type Name string diff --git a/internal/spec/proxy/init.go b/internal/spec/proxy/init.go index 2f0759c..e870eb3 100644 --- a/internal/spec/proxy/init.go +++ b/internal/spec/proxy/init.go @@ -11,7 +11,7 @@ import ( var schema []byte func init() { - if err := spec.Register(NameProxy, schema); err != nil { + if err := spec.Register(string(Name), Version, schema); err != nil { panic(errors.WithStack(err)) } } diff --git a/internal/spec/proxy/schema.json b/internal/spec/proxy/schema.json index 039c94a..c33a407 100644 --- a/internal/spec/proxy/schema.json +++ b/internal/spec/proxy/schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://proxy.emissary.cadoles.com/spec.json", "title": "ProxySpec", "description": "Emissary 'Proxy' specification", @@ -26,16 +26,24 @@ "type": "string" } }, - "required": ["hostPattern", "target"] + "required": [ + "hostPattern", + "target" + ] } } }, - "required": ["address", "mappings"], + "required": [ + "address", + "mappings" + ], "additionalProperties": false } } } }, - "required": ["proxies"], + "required": [ + "proxies" + ], "additionalProperties": false } \ No newline at end of file diff --git a/internal/spec/proxy/spec.go b/internal/spec/proxy/spec.go index 115d03f..0d0a0e4 100644 --- a/internal/spec/proxy/spec.go +++ b/internal/spec/proxy/spec.go @@ -2,7 +2,10 @@ package proxy import "forge.cadoles.com/Cadoles/emissary/internal/spec" -const NameProxy spec.Name = "proxy.emissary.cadoles.com" +const ( + Name string = "proxy.emissary.cadoles.com" + Version = "0.0.0" +) type ID string @@ -21,8 +24,12 @@ type ProxyMapping struct { Target string `json:"target"` } -func (s *Spec) SpecName() spec.Name { - return NameProxy +func (s *Spec) SpecDefinitionName() string { + return Name +} + +func (s *Spec) SpecDefinitionVersion() string { + return Version } func (s *Spec) SpecRevision() int { diff --git a/internal/spec/proxy/validator_test.go b/internal/spec/proxy/validator_test.go index bf80291..6b1f0c6 100644 --- a/internal/spec/proxy/validator_test.go +++ b/internal/spec/proxy/validator_test.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "testing" + "forge.cadoles.com/Cadoles/emissary/internal/datastore/memory" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) @@ -37,11 +38,15 @@ var validatorTestCases = []validatorTestCase{ func TestValidator(t *testing.T) { t.Parallel() - validator := spec.NewValidator() - if err := validator.Register(NameProxy, schema); err != nil { + ctx := context.Background() + + repo := memory.NewSpecDefinitionRepository() + if _, err := repo.Upsert(ctx, Name, Version, schema); err != nil { t.Fatalf("+%v", errors.WithStack(err)) } + validator := spec.NewValidator(repo) + for _, tc := range validatorTestCases { func(tc validatorTestCase) { t.Run(tc.Name, func(t *testing.T) { diff --git a/internal/spec/registry.go b/internal/spec/registry.go new file mode 100644 index 0000000..f857ccc --- /dev/null +++ b/internal/spec/registry.go @@ -0,0 +1,66 @@ +package spec + +import ( + "encoding/json" + + "github.com/pkg/errors" + "github.com/qri-io/jsonschema" +) + +type Registry struct { + definitions map[string]map[string][]byte +} + +func (r *Registry) Register(name string, version string, schema []byte) error { + // Assert that provided schema is valid + if err := json.Unmarshal(schema, &jsonschema.Schema{}); err != nil { + return errors.WithStack(err) + } + + specDefinitions, exists := r.definitions[name] + if !exists { + specDefinitions = make(map[string][]byte) + } + + specDefinitions[version] = schema + + r.definitions[name] = specDefinitions + + return nil +} + +func (r *Registry) Walk(fn func(name string, version string, schema []byte) error) error { + for name, specDefinitions := range r.definitions { + for version, schema := range specDefinitions { + if err := fn(name, version, schema); err != nil { + return errors.WithStack(err) + } + } + } + + return nil +} + +func NewRegistry() *Registry { + return &Registry{ + definitions: make(map[string]map[string][]byte), + } +} + +var defaultRegistry = NewRegistry() + +func Register(name string, version string, schema []byte) error { + if err := defaultRegistry.Register(name, version, schema); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func Walk(fn func(name string, version string, schema []byte) error) error { + if err := defaultRegistry.Walk(fn); err != nil { + return errors.WithStack(err) + } + + return nil +} diff --git a/internal/spec/spec.go b/internal/spec/spec.go index 544ec7b..fdd966f 100644 --- a/internal/spec/spec.go +++ b/internal/spec/spec.go @@ -1,19 +1,27 @@ package spec +const DefaultVersion = "0.0.0" + type Spec interface { - SpecName() Name + SpecDefinitionName() string + SpecDefinitionVersion() string SpecRevision() int SpecData() map[string]any } type RawSpec struct { - Name Name `json:"name"` - Revision int `json:"revision"` - Data map[string]any `json:"data"` + DefinitionName string `json:"name"` + DefinitionVersion string `json:"version"` + Revision int `json:"revision"` + Data map[string]any `json:"data"` } -func (s *RawSpec) SpecName() Name { - return s.Name +func (s *RawSpec) SpecDefinitionName() string { + return s.DefinitionName +} + +func (s *RawSpec) SpecDefinitionVersion() string { + return s.DefinitionVersion } func (s *RawSpec) SpecRevision() int { diff --git a/internal/spec/uci/init.go b/internal/spec/uci/init.go index 348be8a..6cb39f4 100644 --- a/internal/spec/uci/init.go +++ b/internal/spec/uci/init.go @@ -11,7 +11,7 @@ import ( var schema []byte func init() { - if err := spec.Register(NameUCI, schema); err != nil { + if err := spec.Register(string(Name), Version, schema); err != nil { panic(errors.WithStack(err)) } } diff --git a/internal/spec/uci/schema.json b/internal/spec/uci/schema.json index 9119cbd..d42ea05 100644 --- a/internal/spec/uci/schema.json +++ b/internal/spec/uci/schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://uci.emissary.cadoles.com/spec.json", "title": "UCISpec", "description": "Emissary 'UCI' specification", @@ -15,7 +15,9 @@ } } }, - "required": ["packages"], + "required": [ + "packages" + ], "additionalProperties": false }, "postImportCommands": { @@ -33,12 +35,18 @@ } } }, - "required": ["command", "args"], + "required": [ + "command", + "args" + ], "additionalProperties": false } } }, - "required": ["config", "postImportCommands"], + "required": [ + "config", + "postImportCommands" + ], "additionalProperties": false, "$defs": { "package": { @@ -54,7 +62,10 @@ } } }, - "required": ["name", "configs"], + "required": [ + "name", + "configs" + ], "additionalProperties": false }, "config": { @@ -74,11 +85,15 @@ "$ref": "#/$defs/option" } }, - { "type": "null" } + { + "type": "null" + } ] } }, - "required": ["name"], + "required": [ + "name" + ], "additionalProperties": false }, "option": { @@ -86,7 +101,10 @@ "properties": { "type": { "type": "string", - "enum": ["list", "option"] + "enum": [ + "list", + "option" + ] }, "name": { "type": "string" @@ -95,7 +113,11 @@ "type": "string" } }, - "required": ["type", "name", "value"], + "required": [ + "type", + "name", + "value" + ], "additionalProperties": false } } diff --git a/internal/spec/uci/spec.go b/internal/spec/uci/spec.go index 6f56b1b..629b4f9 100644 --- a/internal/spec/uci/spec.go +++ b/internal/spec/uci/spec.go @@ -5,7 +5,10 @@ import ( "forge.cadoles.com/Cadoles/emissary/internal/spec" ) -const NameUCI spec.Name = "uci.emissary.cadoles.com" +const ( + Name string = "uci.emissary.cadoles.com" + Version string = "0.0.0" +) type Spec struct { Revision int `json:"revision"` @@ -18,8 +21,12 @@ type UCIPostImportCommand struct { Args []string `json:"args"` } -func (s *Spec) SpecName() spec.Name { - return NameUCI +func (s *Spec) SpecDefinitionName() string { + return Name +} + +func (s *Spec) SpecDefinitionVersion() string { + return Version } func (s *Spec) SpecRevision() int { diff --git a/internal/spec/uci/validator_test.go b/internal/spec/uci/validator_test.go index d1cfbd3..7770e91 100644 --- a/internal/spec/uci/validator_test.go +++ b/internal/spec/uci/validator_test.go @@ -3,9 +3,10 @@ package uci import ( "context" "encoding/json" - "io/ioutil" + "os" "testing" + "forge.cadoles.com/Cadoles/emissary/internal/datastore/memory" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) @@ -32,17 +33,21 @@ var validatorTestCases = []validatorTestCase{ func TestValidator(t *testing.T) { t.Parallel() - validator := spec.NewValidator() - if err := validator.Register(NameUCI, schema); err != nil { + ctx := context.Background() + + repo := memory.NewSpecDefinitionRepository() + if _, err := repo.Upsert(ctx, Name, Version, schema); err != nil { t.Fatalf("+%v", errors.WithStack(err)) } + validator := spec.NewValidator(repo) + for _, tc := range validatorTestCases { func(tc validatorTestCase) { t.Run(tc.Name, func(t *testing.T) { t.Parallel() - rawSpec, err := ioutil.ReadFile(tc.Source) + rawSpec, err := os.ReadFile(tc.Source) if err != nil { t.Fatalf("+%v", errors.WithStack(err)) } diff --git a/internal/spec/validator.go b/internal/spec/validator.go index 9b54387..e4622ed 100644 --- a/internal/spec/validator.go +++ b/internal/spec/validator.go @@ -4,29 +4,35 @@ import ( "context" "encoding/json" + "forge.cadoles.com/Cadoles/emissary/internal/datastore" "github.com/pkg/errors" "github.com/qri-io/jsonschema" ) type Validator struct { - schemas map[Name]*jsonschema.Schema -} - -func (v *Validator) Register(name Name, rawSchema []byte) error { - schema := &jsonschema.Schema{} - if err := json.Unmarshal(rawSchema, schema); err != nil { - return errors.Wrapf(err, "could not register spec shema '%s'", name) - } - - v.schemas[name] = schema - - return nil + repo datastore.SpecDefinitionRepository } func (v *Validator) Validate(ctx context.Context, spec Spec) error { - schema, exists := v.schemas[spec.SpecName()] - if !exists { - return errors.WithStack(ErrUnknownSchema) + name := spec.SpecDefinitionName() + + version := spec.SpecDefinitionVersion() + if version == "" { + version = DefaultVersion + } + + specDef, err := v.repo.Get(ctx, name, version) + if err != nil { + if errors.Is(err, datastore.ErrNotFound) { + return errors.WithStack(ErrUnknownSchema) + } + + return errors.WithStack(err) + } + + schema := &jsonschema.Schema{} + if err := json.Unmarshal(specDef.Schema, schema); err != nil { + return errors.WithStack(err) } state := schema.Validate(ctx, map[string]any(spec.SpecData())) @@ -37,26 +43,8 @@ func (v *Validator) Validate(ctx context.Context, spec Spec) error { return nil } -func NewValidator() *Validator { +func NewValidator(repo datastore.SpecDefinitionRepository) *Validator { return &Validator{ - schemas: make(map[Name]*jsonschema.Schema), + repo: repo, } } - -var defaultValidator = NewValidator() - -func Register(name Name, rawSchema []byte) error { - if err := defaultValidator.Register(name, rawSchema); err != nil { - return errors.WithStack(err) - } - - return nil -} - -func Validate(ctx context.Context, spec Spec) error { - if err := defaultValidator.Validate(ctx, spec); err != nil { - return errors.WithStack(err) - } - - return nil -} diff --git a/migrations/sqlite/0000004_spec_definition.down.sql b/migrations/sqlite/0000004_spec_definition.down.sql new file mode 100644 index 0000000..1bb19a7 --- /dev/null +++ b/migrations/sqlite/0000004_spec_definition.down.sql @@ -0,0 +1 @@ +DROP TABLE spec_definitions; \ No newline at end of file diff --git a/migrations/sqlite/0000004_spec_definition.up.sql b/migrations/sqlite/0000004_spec_definition.up.sql new file mode 100644 index 0000000..a320406 --- /dev/null +++ b/migrations/sqlite/0000004_spec_definition.up.sql @@ -0,0 +1,9 @@ +CREATE TABLE + spec_definitions ( + name TEXT NOT NULL, + version TEXT NOT NULL, + schema TEXT NOT NULL, + created_at datetime NOT NULL, + updated_at datetime NOT NULL, + UNIQUE (name, version) + ); \ No newline at end of file diff --git a/migrations/sqlite/0000005_spec_version.down.sql b/migrations/sqlite/0000005_spec_version.down.sql new file mode 100644 index 0000000..f83cacc --- /dev/null +++ b/migrations/sqlite/0000005_spec_version.down.sql @@ -0,0 +1,35 @@ +ALTER TABLE specs +RENAME TO _specs; + +CREATE TABLE + specs ( + id INTEGER PRIMARY KEY, + agent_id INTEGER, + tenant_id TEXT, + name TEXT NOT NULL, + revision INTEGER DEFAULT 0, + data TEXT, + created_at datetime NOT NULL, + updated_at datetime NOT NULL, + FOREIGN KEY (tenant_id) REFERENCES tenants (id), + FOREIGN KEY (agent_id) REFERENCES agents (id) ON DELETE CASCADE, + UNIQUE (agent_id, name) ON CONFLICT REPLACE + ); + +INSERT INTO + specs +SELECT + id, + agent_id, + tenant_id, + name, + revision, + data, + created_at, + updated_at +FROM + _specs; + +DROP TABLE _specs; + +--- \ No newline at end of file diff --git a/migrations/sqlite/0000005_spec_version.up.sql b/migrations/sqlite/0000005_spec_version.up.sql new file mode 100644 index 0000000..614360e --- /dev/null +++ b/migrations/sqlite/0000005_spec_version.up.sql @@ -0,0 +1,36 @@ +-- Add unique constraint on name/version to specs +ALTER TABLE specs +RENAME TO _specs; + +CREATE TABLE + specs ( + id INTEGER PRIMARY KEY, + agent_id INTEGER, + tenant_id TEXT, + name TEXT NOT NULL, + version TEXT NOT NULL, + revision INTEGER DEFAULT 0, + data TEXT, + created_at datetime NOT NULL, + updated_at datetime NOT NULL, + FOREIGN KEY (tenant_id) REFERENCES tenants (id), + FOREIGN KEY (agent_id) REFERENCES agents (id) ON DELETE CASCADE, + UNIQUE (agent_id, name, version) ON CONFLICT REPLACE + ); + +INSERT INTO + specs +SELECT + id, + agent_id, + tenant_id, + name, + "0.0.0", + revision, + data, + created_at, + updated_at +FROM + _specs; + +DROP TABLE _specs; \ No newline at end of file diff --git a/pkg/client/alias.go b/pkg/client/alias.go index 0fef79c..7a92af2 100644 --- a/pkg/client/alias.go +++ b/pkg/client/alias.go @@ -8,8 +8,7 @@ import ( ) type ( - Spec = spec.Spec - SpecName = spec.Name + Spec = spec.Spec ) type ( diff --git a/pkg/client/delete_agent_spec.go b/pkg/client/delete_agent_spec.go index be479e8..d931605 100644 --- a/pkg/client/delete_agent_spec.go +++ b/pkg/client/delete_agent_spec.go @@ -4,30 +4,32 @@ import ( "context" "fmt" - "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) -func (c *Client) DeleteAgentSpec(ctx context.Context, agentID AgentID, name SpecName, funcs ...OptionFunc) (SpecName, error) { +func (c *Client) DeleteAgentSpec(ctx context.Context, agentID AgentID, name string, version string, funcs ...OptionFunc) (string, string, error) { payload := struct { - Name spec.Name `json:"name"` + Name string `json:"name"` + Version string `json:"version"` }{ - Name: name, + Name: name, + Version: version, } response := withResponse[struct { - Name spec.Name `json:"name"` + Name string `json:"name"` + Version string `json:"version"` }]() path := fmt.Sprintf("/api/v1/agents/%d/specs", agentID) if err := c.apiDelete(ctx, path, payload, &response, funcs...); err != nil { - return "", errors.WithStack(err) + return "", "", errors.WithStack(err) } if response.Error != nil { - return "", errors.WithStack(response.Error) + return "", "", errors.WithStack(response.Error) } - return response.Data.Name, nil + return response.Data.Name, response.Data.Version, nil } diff --git a/pkg/client/get_agent_specs.go b/pkg/client/get_agent_specs.go index fd42b2c..a855375 100644 --- a/pkg/client/get_agent_specs.go +++ b/pkg/client/get_agent_specs.go @@ -4,13 +4,14 @@ import ( "context" "fmt" + "forge.cadoles.com/Cadoles/emissary/internal/datastore" "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) func (c *Client) GetAgentSpecs(ctx context.Context, agentID AgentID, funcs ...OptionFunc) ([]Spec, error) { response := withResponse[struct { - Specs []*spec.RawSpec `json:"specs"` + Specs []*datastore.Spec `json:"specs"` }]() path := fmt.Sprintf("/api/v1/agents/%d/specs", agentID) diff --git a/pkg/client/update_agent_spec.go b/pkg/client/update_agent_spec.go index 86f9c84..1170171 100644 --- a/pkg/client/update_agent_spec.go +++ b/pkg/client/update_agent_spec.go @@ -6,17 +6,18 @@ import ( "forge.cadoles.com/Cadoles/emissary/internal/agent/metadata" "forge.cadoles.com/Cadoles/emissary/internal/datastore" - "forge.cadoles.com/Cadoles/emissary/internal/spec" "github.com/pkg/errors" ) func (c *Client) UpdateAgentSpec(ctx context.Context, agentID AgentID, spc Spec, funcs ...OptionFunc) (Spec, error) { payload := struct { - Name spec.Name `json:"name"` + Name string `json:"name"` + Version string `json:"version"` Revision int `json:"revision"` Data metadata.Metadata `json:"data"` }{ - Name: spc.SpecName(), + Name: spc.SpecDefinitionName(), + Version: spc.SpecDefinitionVersion(), Revision: spc.SpecRevision(), Data: spc.SpecData(), }