arcad/emissary
arcad
/
emissary
16
0
Fork 0
Code Issues 14 Pull Requests Packages Projects Releases 3 Wiki Activity

feat: agent metadata with custom collectors

This commit is contained in:
wpetit 2023-03-02 13:05:24 +01:00
parent 3310c09320
commit 1ff29ae1fb
40 changed files with 998 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -6,3 +6,4 @@ dist/
/state.json
/emissary.sqlite
/.gitea-release
/agent-key.json

15
go.mod
View File

@ -8,12 +8,15 @@ require (
github.com/btcsuite/btcd/btcutil v1.1.3
github.com/davecgh/go-spew v1.1.1
github.com/denisbrodbeck/machineid v1.0.1
github.com/evanphx/json-patch/v5 v5.6.0
github.com/go-chi/chi v4.1.2+incompatible
github.com/go-chi/cors v1.2.1
github.com/golang-migrate/migrate/v4 v4.15.2
github.com/jackc/pgx/v5 v5.2.0
github.com/jedib0t/go-pretty/v6 v6.4.4
github.com/mitchellh/mapstructure v1.5.0
github.com/pkg/errors v0.9.1
github.com/qri-io/jsonschema v0.2.1
github.com/santhosh-tekuri/jsonschema/v5 v5.1.1
github.com/urfave/cli/v2 v2.23.7
gitlab.com/wpetit/goweb v0.0.0-20230227162855-a1f09bafccb3
@ -27,12 +30,13 @@ require (
cdr.dev/slog v1.4.1 // indirect
github.com/alecthomas/chroma v0.10.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/dlclark/regexp2 v1.8.0 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
@ -40,15 +44,19 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.1.2 // indirect
github.com/jedib0t/go-pretty/v6 v6.4.4 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/leodido/go-urn v1.2.2 // indirect
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc v1.0.4 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/jwx/v2 v2.0.8 // indirect
github.com/lestrrat-go/option v1.0.0 // indirect
github.com/lib/pq v1.10.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/qri-io/jsonpointer v0.1.1 // indirect
github.com/qri-io/jsonschema v0.2.1 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
@ -66,7 +74,6 @@ require (
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/evanphx/json-patch.v5 v5.6.0 // indirect
gopkg.in/go-playground/validator.v9 v9.31.0 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect

25
go.sum
View File

@ -395,6 +395,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218=
github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMSRhl4D7AQ=
github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI=
@ -552,6 +554,8 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
@ -854,10 +858,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/leodido/go-urn v1.2.2 h1:7z68G0FCGvDk646jz1AelTYNYWrTNm0bEcFAo147wt4=
github.com/leodido/go-urn v1.2.2/go.mod h1:kUaIbLZWttglzwNuG0pgsh5vuV6u2YcGBYz1hIPjtOQ=
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
github.com/lestrrat-go/httprc v1.0.4 h1:bAZymwoZQb+Oq8MEbyipag7iSq6YIga8Wj6GOiJGdI8=
github.com/lestrrat-go/httprc v1.0.4/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
github.com/lestrrat-go/jwx/v2 v2.0.8 h1:jCFT8oc0hEDVjgUgsBy1F9cbjsjAVZSXNi7JaU9HR/Q=
github.com/lestrrat-go/jwx/v2 v2.0.8/go.mod h1:zLxnyv9rTlEvOUHbc48FAfIL8iYu2hHvIRaTFGc8mT0=
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@ -1170,7 +1184,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@ -1226,8 +1239,6 @@ github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPS
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
gitlab.com/wpetit/goweb v0.0.0-20230206085656-dec695f0e2e9 h1:6JlkcdjYVQglPWYuemK2MoZAtRE4vFx85zLXflGIyI8=
gitlab.com/wpetit/goweb v0.0.0-20230206085656-dec695f0e2e9/go.mod h1:3sus4zjoUv1GB7eDLL60QaPkUnXJCWBpjvbe0jWifeY=
gitlab.com/wpetit/goweb v0.0.0-20230227162855-a1f09bafccb3 h1:ddXRTeqEr7LcHQEtkd6gogZOh9tI1Y6Gappr0a1oa2I=
gitlab.com/wpetit/goweb v0.0.0-20230227162855-a1f09bafccb3/go.mod h1:3sus4zjoUv1GB7eDLL60QaPkUnXJCWBpjvbe0jWifeY=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@ -1315,6 +1326,7 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@ -1430,6 +1442,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220111093109-d55c255bac03/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@ -1889,8 +1902,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/evanphx/json-patch.v5 v5.6.0 h1:BMT6KIwBD9CaU91PJCZIe46bDmBWa9ynTQgJIOpfQBk=
gopkg.in/evanphx/json-patch.v5 v5.6.0/go.mod h1:/kvTRh1TVm5wuM6OkHxqXtE/1nUZZpihg29RtuIyfvk=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=

73
internal/agent/agent.go
View File

@ -4,13 +4,21 @@ import (
"context"
"time"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/client"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/api"
"gitlab.com/wpetit/goweb/logger"
)
type Agent struct {
thumbprint string
privateKey jwk.Key
client *client.Client
controllers []Controller
interval time.Duration
collectors []metadata.Collector
}
func (a *Agent) Run(ctx context.Context) error {
@ -21,10 +29,20 @@ func (a *Agent) Run(ctx context.Context) error {
ticker := time.NewTicker(a.interval)
defer ticker.Stop()
ctx = withClient(ctx, a.client)
for {
select {
case <-ticker.C:
logger.Debug(ctx, "registering agent")
if err := a.registerAgent(ctx, state); err != nil {
logger.Error(ctx, "could not register agent", logger.E(errors.WithStack(err)))
continue
}
logger.Debug(ctx, "state before reconciliation", logger.F("state", state))
if err := a.Reconcile(ctx, state); err != nil {
@ -58,14 +76,67 @@ func (a *Agent) Reconcile(ctx context.Context, state *State) error {
return nil
}
func New(funcs ...OptionFunc) *Agent {
func (a *Agent) registerAgent(ctx context.Context, state *State) error {
meta, err := a.collectMetadata(ctx)
if err != nil {
return errors.WithStack(err)
}
sorted := metadata.Sort(meta)
agent, err := a.client.RegisterAgent(ctx, a.privateKey, a.thumbprint, sorted)
if err != nil {
return errors.WithStack(err)
}
state.agentID = agent.ID
return nil
}
func (a *Agent) collectMetadata(ctx context.Context) (map[string]any, error) {
metadata := make(map[string]any)
for _, collector := range a.collectors {
name, value, err := collector.Collect(ctx)
if err != nil {
logger.Error(
ctx, "could not collect metadata",
logger.E(errors.WithStack(err)), logger.F("name", name),
)
continue
}
metadata[name] = value
}
return metadata, nil
}
func isAPIError(err error, code api.ErrorCode) (bool, any) {
apiError := &api.Error{}
if errors.As(err, &apiError) && apiError.Code == code {
return true, apiError.Data
}
return false, nil
}
func New(serverURL string, privateKey jwk.Key, thumbprint string, funcs ...OptionFunc) *Agent {
opt := defaultOption()
for _, fn := range funcs {
fn(opt)
}
client := client.New(serverURL)
return &Agent{
privateKey: privateKey,
thumbprint: thumbprint,
client: client,
controllers: opt.Controllers,
interval: opt.Interval,
collectors: opt.Collectors,
}
}

27
internal/agent/context.go Normal file
View File

@ -0,0 +1,27 @@
package agent
import (
"context"
"forge.cadoles.com/Cadoles/emissary/internal/client"
"github.com/pkg/errors"
)
type contextKey string
const (
contextKeyClient contextKey = "client"
)
func withClient(ctx context.Context, client *client.Client) context.Context {
return context.WithValue(ctx, contextKeyClient, client)
}
func Client(ctx context.Context) *client.Client {
client, ok := ctx.Value(contextKeyClient).(*client.Client)
if !ok {
panic(errors.New("could not retrieve client from context"))
}
return client
}

59
internal/agent/controller/spec/controller.go
View File

@ -4,18 +4,13 @@ import (
"context"
"forge.cadoles.com/Cadoles/emissary/internal/agent"
"forge.cadoles.com/Cadoles/emissary/internal/agent/machineid"
"forge.cadoles.com/Cadoles/emissary/internal/client"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"forge.cadoles.com/Cadoles/emissary/internal/server"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/api"
"gitlab.com/wpetit/goweb/logger"
)
type Controller struct {
client *client.Client
}
type Controller struct{}
// Name implements node.Controller.
func (c *Controller) Name() string {
@ -24,22 +19,12 @@ func (c *Controller) Name() string {
// Reconcile implements node.Controller.
func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error {
machineID, err := machineid.Get()
if err != nil {
return errors.WithStack(err)
}
cl := agent.Client(ctx)
ctx = logger.With(ctx, logger.F("machineID", machineID))
agent, err := c.client.RegisterAgent(ctx, machineID)
isAlreadyRegisteredErr, _ := isAPIError(err, server.ErrCodeAlreadyRegistered)
switch {
case isAlreadyRegisteredErr:
agents, _, err := c.client.QueryAgents(
agents, _, err := cl.QueryAgents(
ctx,
client.WithQueryAgentsLimit(1),
client.WithQueryAgentsRemoteID(machineID),
client.WithQueryAgentsID(state.AgentID()),
)
if err != nil {
return errors.WithStack(err)
@ -51,29 +36,14 @@ func (c *Controller) Reconcile(ctx context.Context, state *agent.State) error {
return nil
}
if err := c.reconcileAgent(ctx, state, agents[0]); err != nil {
if err := c.reconcileAgent(ctx, cl, state, agents[0]); err != nil {
return errors.WithStack(err)
}
return nil
case agent != nil:
if err := c.reconcileAgent(ctx, state, agent); err != nil {
return errors.WithStack(err)
}
return nil
case err != nil:
logger.Error(ctx, "could not contact server", logger.E(errors.WithStack(err)))
return nil
}
return nil
}
func (c *Controller) reconcileAgent(ctx context.Context, state *agent.State, agent *datastore.Agent) error {
func (c *Controller) reconcileAgent(ctx context.Context, client *client.Client, state *agent.State, agent *datastore.Agent) error {
ctx = logger.With(ctx, logger.F("agentID", agent.ID))
if agent.Status != datastore.AgentStatusAccepted {
@ -82,7 +52,7 @@ func (c *Controller) reconcileAgent(ctx context.Context, state *agent.State, age
return nil
}
specs, err := c.client.GetAgentSpecs(ctx, agent.ID)
specs, err := client.GetAgentSpecs(ctx, agent.ID)
if err != nil {
logger.Error(ctx, "could not retrieve agent specs", logger.E(errors.WithStack(err)))
@ -98,19 +68,8 @@ func (c *Controller) reconcileAgent(ctx context.Context, state *agent.State, age
return nil
}
func NewController(serverURL string) *Controller {
client := client.New(serverURL)
return &Controller{client}
}
func isAPIError(err error, code api.ErrorCode) (bool, any) {
apiError := &api.Error{}
if errors.As(err, &apiError) && apiError.Code == code {
return true, apiError.Data
}
return false, nil
func NewController() *Controller {
return &Controller{}
}
var _ agent.Controller = &Controller{}

12
internal/agent/metadata/collector.go Normal file
View File

@ -0,0 +1,12 @@
package metadata
import (
"context"
"errors"
)
var ErrMetadataNotAvailable = errors.New("metadata not available")
type Collector interface {
Collect(context.Context) (string, string, error)
}

31
internal/agent/metadata/collector/buildinfo/collector.go Normal file
View File

@ -0,0 +1,31 @@
package buildinfo
import (
"context"
"runtime/debug"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"github.com/pkg/errors"
)
const (
MetadataBuildInfo = "buildinfo"
)
type Collector struct{}
// Collect implements agent.MetadataCollector
func (c *Collector) Collect(ctx context.Context) (string, string, error) {
buildInfo, ok := debug.ReadBuildInfo()
if !ok {
return "", "", errors.WithStack(metadata.ErrMetadataNotAvailable)
}
return MetadataBuildInfo, buildInfo.String(), nil
}
func NewCollector() *Collector {
return &Collector{}
}
var _ metadata.Collector = &Collector{}

46
internal/agent/metadata/collector/shell/collector.go Normal file
View File

@ -0,0 +1,46 @@
package shell
import (
"bytes"
"context"
"os"
"os/exec"
"strings"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"github.com/pkg/errors"
)
type Collector struct {
name string
command string
args []string
}
// Collect implements agent.MetadataCollector
func (c *Collector) Collect(ctx context.Context) (string, string, error) {
cmd := exec.CommandContext(ctx, c.command, c.args...)
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
return "", "", errors.WithStack(err)
}
value := strings.TrimSpace(buf.String())
return c.name, value, nil
}
func NewCollector(name string, command string, args ...string) *Collector {
return &Collector{
name: name,
command: command,
args: args,
}
}
var _ metadata.Collector = &Collector{}

3
internal/agent/metadata/metadata.go Normal file
View File

@ -0,0 +1,3 @@
package metadata
type Metadata map[string]any

37
internal/agent/metadata/sort.go Normal file
View File

@ -0,0 +1,37 @@
package metadata
import (
"sort"
)
type Tuple struct {
Key string `json:"key"`
Value any `json:"value"`
}
func Sort(metadata map[string]any) []Tuple {
keys := make([]string, 0, len(metadata))
for k := range metadata {
keys = append(keys, k)
}
sort.Strings(keys)
tuples := make([]Tuple, len(keys))
for i, k := range keys {
tuples[i] = Tuple{k, metadata[k]}
}
return tuples
}
func FromSorted(tuples []Tuple) map[string]any {
metadata := make(map[string]any)
for _, t := range tuples {
metadata[t.Key] = t.Value
}
return metadata
}

31
internal/agent/option.go
View File

@ -1,31 +0,0 @@
package agent
import (
"time"
)
type Option struct {
Interval time.Duration
Controllers []Controller
}
type OptionFunc func(*Option)
func defaultOption() *Option {
return &Option{
Controllers: make([]Controller, 0),
Interval: 10 * time.Second,
}
}
func WithControllers(controllers ...Controller) OptionFunc {
return func(opt *Option) {
opt.Controllers = controllers
}
}
func WithInterval(interval time.Duration) OptionFunc {
return func(opt *Option) {
opt.Interval = interval
}
}

43
internal/agent/options.go Normal file
View File

@ -0,0 +1,43 @@
package agent
import (
"time"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/client"
)
type Options struct {
Client *client.Client
Interval time.Duration
Controllers []Controller
Collectors []metadata.Collector
}
type OptionFunc func(*Options)
func defaultOption() *Options {
return &Options{
Controllers: make([]Controller, 0),
Interval: 10 * time.Second,
Collectors: make([]metadata.Collector, 0),
}
}
func WithControllers(controllers ...Controller) OptionFunc {
return func(opt *Options) {
opt.Controllers = controllers
}
}
func WithInterval(interval time.Duration) OptionFunc {
return func(opt *Options) {
opt.Interval = interval
}
}
func WithCollectors(collectors ...metadata.Collector) OptionFunc {
return func(opts *Options) {
opts.Collectors = collectors
}
}

11
internal/agent/state.go
View File

@ -3,6 +3,7 @@ package agent
import (
"encoding/json"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"forge.cadoles.com/Cadoles/emissary/internal/spec"
"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"
@ -13,7 +14,8 @@ var ErrSpecNotFound = errors.New("spec not found")
type Specs map[spec.Name]spec.Spec
type State struct {
specs Specs `json:"specs"`
agentID datastore.AgentID
specs Specs
}
func NewState() *State {
@ -24,8 +26,10 @@ func NewState() *State {
func (s *State) MarshalJSON() ([]byte, error) {
state := struct {
ID datastore.AgentID `json:"agentId"`
Specs map[spec.Name]*spec.RawSpec `json:"specs"`
}{
ID: s.agentID,
Specs: func(specs map[spec.Name]spec.Spec) map[spec.Name]*spec.RawSpec {
rawSpecs := make(map[spec.Name]*spec.RawSpec)
@ -51,6 +55,7 @@ func (s *State) MarshalJSON() ([]byte, error) {
func (s *State) UnmarshalJSON(data []byte) error {
state := struct {
AgentID datastore.AgentID `json:"agentId"`
Specs map[spec.Name]*spec.RawSpec `json:"specs"`
}{}
@ -71,6 +76,10 @@ func (s *State) UnmarshalJSON(data []byte) error {
return nil
}
func (s *State) AgentID() datastore.AgentID {
return s.agentID
}
func (s *State) Specs() Specs {
return s.specs
}

1
internal/client/client.go
View File

@ -14,6 +14,7 @@ import (
type Client struct {
http *http.Client
token string
serverURL string
}

12
internal/client/query_agents.go
View File

@ -14,7 +14,7 @@ type QueryAgentsOptionFunc func(*QueryAgentsOptions)
type QueryAgentsOptions struct {
Limit *int
Offset *int
RemoteIDs []string
Thumbprints []string
IDs []datastore.AgentID
Statuses []datastore.AgentStatus
}
@ -31,9 +31,9 @@ func WithQueryAgentsOffset(offset int) QueryAgentsOptionFunc {
}
}
func WithQueryAgentsRemoteID(remoteIDs ...string) QueryAgentsOptionFunc {
func WithQueryAgentsThumbprints(thumbprints ...string) QueryAgentsOptionFunc {
return func(opts *QueryAgentsOptions) {
opts.RemoteIDs = remoteIDs
opts.Thumbprints = thumbprints
}
}
@ -61,11 +61,11 @@ func (c *Client) QueryAgents(ctx context.Context, funcs ...QueryAgentsOptionFunc
query.Set("ids", joinSlice(options.IDs))
}
if options.RemoteIDs != nil && len(options.RemoteIDs) > 0 {
query.Set("remoteIds", joinSlice(options.RemoteIDs))
if options.Thumbprints != nil && len(options.Thumbprints) > 0 {
query.Set("thumbprints", joinSlice(options.Thumbprints))
}
if options.Statuses != nil && len(options.RemoteIDs) > 0 {
if options.Statuses != nil && len(options.Statuses) > 0 {
query.Set("statuses", joinSlice(options.Statuses))
}

24
internal/client/register_agent.go
View File

@ -3,15 +3,33 @@ package client
import (
"context"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"github.com/pkg/errors"
)
func (c *Client) RegisterAgent(ctx context.Context, remoteID string) (*datastore.Agent, error) {
func (c *Client) RegisterAgent(ctx context.Context, key jwk.Key, thumbprint string, meta []metadata.Tuple) (*datastore.Agent, error) {
keySet, err := jwk.PublicKeySet(key)
if err != nil {
return nil, errors.WithStack(err)
}
signature, err := jwk.Sign(key, thumbprint, meta)
if err != nil {
return nil, errors.WithStack(err)
}
payload := struct {
RemoteID string `json:"remoteId"`
KeySet jwk.Set `json:"keySet"`
Thumbprint string `json:"thumbprint"`
Metadata []metadata.Tuple `json:"metadata"`
Signature string `json:"signature"`
}{
RemoteID: remoteID,
Thumbprint: thumbprint,
Metadata: meta,
Signature: signature,
KeySet: keySet,
}
response := withResponse[struct {

11
internal/client/update_agent_spec.go
View File

@ -4,20 +4,21 @@ import (
"context"
"fmt"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"forge.cadoles.com/Cadoles/emissary/internal/spec"
"github.com/pkg/errors"
)
func (c *Client) UpdateAgentSpec(ctx context.Context, agentID datastore.AgentID, name spec.Name, revision int, data any) (*datastore.Spec, error) {
func (c *Client) UpdateAgentSpec(ctx context.Context, agentID datastore.AgentID, spc spec.Spec) (*datastore.Spec, error) {
payload := struct {
Name spec.Name `json:"name"`
Revision int `json:"revision"`
Data any `json:"data"`
Data metadata.Metadata `json:"data"`
}{
Name: name,
Revision: revision,
Data: data,
Name: spc.SpecName(),
Revision: spc.SpecRevision(),
Data: spc.SpecData(),
}
response := withResponse[struct {

37
internal/command/agent/run.go
View File

@ -8,7 +8,13 @@ import (
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/openwrt"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/persistence"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/spec"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata/collector/buildinfo"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata/collector/shell"
"forge.cadoles.com/Cadoles/emissary/internal/command/common"
"forge.cadoles.com/Cadoles/emissary/internal/config"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"forge.cadoles.com/Cadoles/emissary/internal/machineid"
"github.com/pkg/errors"
_ "github.com/santhosh-tekuri/jsonschema/v5/httploader"
"github.com/urfave/cli/v2"
@ -40,7 +46,7 @@ func RunCommand() *cli.Command {
}
if ctrlConf.Spec.Enabled {
controllers = append(controllers, spec.NewController(string(ctrlConf.Spec.ServerURL)))
controllers = append(controllers, spec.NewController())
}
if ctrlConf.Gateway.Enabled {
@ -53,9 +59,26 @@ func RunCommand() *cli.Command {
))
}
key, err := jwk.LoadOrGenerate(string(conf.Agent.PrivateKeyPath), jwk.DefaultKeySize)
if err != nil {
return errors.WithStack(err)
}
thumbprint, err := machineid.Get()
if err != nil {
return errors.WithStack(err)
}
collectors := createShellCollectors(&conf.Agent)
collectors = append(collectors, buildinfo.NewCollector())
agent := agent.New(
string(conf.Agent.ServerURL),
key,
thumbprint,
agent.WithInterval(time.Duration(conf.Agent.ReconciliationInterval)*time.Second),
agent.WithControllers(controllers...),
agent.WithCollectors(collectors...),
)
if err := agent.Run(ctx.Context); err != nil {
@ -66,3 +89,15 @@ func RunCommand() *cli.Command {
},
}
}
func createShellCollectors(conf *config.AgentConfig) []metadata.Collector {
collectors := make([]metadata.Collector, 0)
for _, c := range conf.Collectors {
collector := shell.NewCollector(string(c.Name), string(c.Command), c.Args...)
collectors = append(collectors, collector)
}
return collectors
}

44
internal/command/client/agent/get.go Normal file
View File

@ -0,0 +1,44 @@
package agent
import (
"os"
"forge.cadoles.com/Cadoles/emissary/internal/client"
agentFlag "forge.cadoles.com/Cadoles/emissary/internal/command/client/agent/flag"
"forge.cadoles.com/Cadoles/emissary/internal/command/client/apierr"
clientFlag "forge.cadoles.com/Cadoles/emissary/internal/command/client/flag"
"forge.cadoles.com/Cadoles/emissary/internal/format"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
)
func GetCommand() *cli.Command {
return &cli.Command{
Name: "get",
Usage: "Get agent",
Flags: agentFlag.WithAgentFlags(),
Action: func(ctx *cli.Context) error {
baseFlags := clientFlag.GetBaseFlags(ctx)
agentID, err := agentFlag.AssertAgentID(ctx)
if err != nil {
return errors.WithStack(err)
}
client := client.New(baseFlags.ServerURL)
agent, err := client.GetAgent(ctx.Context, agentID)
if err != nil {
return errors.WithStack(apierr.Wrap(err))
}
hints := agentHints(baseFlags.OutputMode)
if err := format.Write(baseFlags.Format, os.Stdout, hints, agent); err != nil {
return errors.WithStack(err)
}
return nil
},
}
}

4
internal/command/client/agent/query.go
View File

@ -25,9 +25,7 @@ func QueryCommand() *cli.Command {
return errors.WithStack(apierr.Wrap(err))
}
hints := format.Hints{
OutputMode: baseFlags.OutputMode,
}
hints := agentHints(baseFlags.OutputMode)
if err := format.Write(baseFlags.Format, os.Stdout, hints, clientFlag.AsAnySlice(agents)...); err != nil {
return errors.WithStack(err)

1
internal/command/client/agent/root.go
View File

@ -13,6 +13,7 @@ func Root() *cli.Command {
QueryCommand(),
CountCommand(),
UpdateCommand(),
GetCommand(),
spec.Root(),
},
}

29
internal/command/client/agent/spec/update.go
View File

@ -26,7 +26,7 @@ func UpdateCommand() *cli.Command {
},
&cli.StringFlag{
Name: "spec-data",
Usage: "use `DATA` as spec data",
Usage: "use `DATA` as spec data, '-' to read from STDIN",
},
&cli.BoolFlag{
Name: "no-patch",
@ -94,7 +94,17 @@ func UpdateCommand() *cli.Command {
revision = specificRevision
}
spec, err := client.UpdateAgentSpec(ctx.Context, agentID, specName, revision, specData)
rawSpec := &spec.RawSpec{
Name: specName,
Revision: revision,
Data: specData,
}
if err := spec.Validate(ctx.Context, rawSpec); err != nil {
return errors.WithStack(apierr.Wrap(err))
}
spec, err := client.UpdateAgentSpec(ctx.Context, agentID, rawSpec)
if err != nil {
return errors.WithStack(apierr.Wrap(err))
}
@ -122,23 +132,30 @@ func assertSpecName(ctx *cli.Context) (spec.Name, error) {
return spec.Name(specName), nil
}
func assertSpecData(ctx *cli.Context) (any, error) {
func assertSpecData(ctx *cli.Context) (map[string]any, error) {
rawSpecData := ctx.String("spec-data")
if rawSpecData == "" {
return nil, errors.New("flag 'spec-data' is required")
}
var specData any
var specData map[string]any
if rawSpecData == "-" {
decoder := json.NewDecoder(os.Stdin)
if err := decoder.Decode(&specData); err != nil {
return nil, errors.WithStack(err)
}
} else {
if err := json.Unmarshal([]byte(rawSpecData), &specData); err != nil {
return nil, errors.WithStack(err)
}
}
return specData, nil
}
func applyPatch(origin any, patch any) (any, error) {
func applyPatch(origin any, patch any) (map[string]any, error) {
originJSON, err := json.Marshal(origin)
if err != nil {
return nil, errors.WithStack(err)
@ -154,7 +171,7 @@ func applyPatch(origin any, patch any) (any, error) {
return nil, errors.WithStack(err)
}
var specData any
var specData map[string]any
if err := json.Unmarshal(result, &specData); err != nil {
}

4
internal/command/client/agent/update.go
View File

@ -45,9 +45,7 @@ func UpdateCommand() *cli.Command {
return errors.WithStack(apierr.Wrap(err))
}
hints := format.Hints{
OutputMode: baseFlags.OutputMode,
}
hints := agentHints(baseFlags.OutputMode)
if err := format.Write(baseFlags.Format, os.Stdout, hints, agent); err != nil {
return errors.WithStack(err)

16
internal/command/client/agent/util.go Normal file
View File

@ -0,0 +1,16 @@
package agent
import "forge.cadoles.com/Cadoles/emissary/internal/format"
func agentHints(outputMode format.OutputMode) format.Hints {
return format.Hints{
OutputMode: outputMode,
Props: []format.Prop{
format.NewProp("ID", "ID"),
format.NewProp("Thumbprint", "Thumbprint"),
format.NewProp("Status", "Status"),
format.NewProp("CreatedAt", "CreatedAt"),
format.NewProp("UpdatedAt", "UpdatedAt"),
},
}
}

21
internal/config/agent.go
View File

@ -1,8 +1,17 @@
package config
type AgentConfig struct {
ServerURL InterpolatedString `yaml:"serverUrl"`
PrivateKeyPath InterpolatedString `yaml:"privateKeyPath"`
ReconciliationInterval InterpolatedInt `yaml:"reconciliationInterval"`
Controllers ControllersConfig `yaml:"controllers"`
Collectors []ShellCollectorConfig `yaml:"collectors"`
}
type ShellCollectorConfig struct {
Name InterpolatedString `yaml:"name"`
Command InterpolatedString `yaml:"command"`
Args InterpolatedStringSlice `yaml:"args"`
}
type ControllersConfig struct {
@ -19,9 +28,7 @@ type PersistenceControllerConfig struct {
type SpecControllerConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
ServerURL InterpolatedString `yaml:"serverUrl"`
}
type GatewayControllerConfig struct {
Enabled InterpolatedBool `yaml:"enabled"`
}
@ -34,11 +41,12 @@ type UCIControllerConfig struct {
func NewDefaultAgentConfig() AgentConfig {
return AgentConfig{
ServerURL: "http://127.0.0.1:3000",
PrivateKeyPath: "agent-key.json",
ReconciliationInterval: 5,
Controllers: ControllersConfig{
Spec: SpecControllerConfig{
Enabled: true,
ServerURL: "http://127.0.0.1:3000",
},
Persistence: PersistenceControllerConfig{
Enabled: true,
@ -53,5 +61,12 @@ func NewDefaultAgentConfig() AgentConfig {
BinPath: "uci",
},
},
Collectors: []ShellCollectorConfig{
{
Name: "uname",
Command: "uname",
Args: []string{"-a"},
},
},
}
}

33
internal/datastore/agent.go
View File

@ -1,7 +1,11 @@
package datastore
import (
"encoding/json"
"time"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/pkg/errors"
)
type AgentID int64
@ -17,8 +21,35 @@ const (
type Agent struct {
ID AgentID `json:"id"`
RemoteID string `json:"remoteId"`
Thumbprint string `json:"thumbprint"`
KeySet *SerializableKeySet `json:"keyset,omitempty"`
Metadata map[string]any `json:"metadata,omitempty"`
Status AgentStatus `json:"status"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type SerializableKeySet struct {
jwk.Set
}
func (s *SerializableKeySet) UnmarshalJSON(data []byte) error {
keySet := jwk.NewSet()
if err := json.Unmarshal(data, &keySet); err != nil {
return errors.WithStack(err)
}
s.Set = keySet
return nil
}
func (s *SerializableKeySet) MarshalJSON() ([]byte, error) {
data, err := json.Marshal(s.Set)
if err != nil {
return nil, errors.WithStack(err)
}
return data, nil
}

42
internal/datastore/agent_repository.go
View File

@ -1,9 +1,13 @@
package datastore
import "context"
import (
"context"
"github.com/lestrrat-go/jwx/v2/jwk"
)
type AgentRepository interface {
Create(ctx context.Context, remoteID string, state AgentStatus) (*Agent, error)
Create(ctx context.Context, thumbprint string, keySet jwk.Set, metadata map[string]any) (*Agent, error)
Get(ctx context.Context, id AgentID) (*Agent, error)
Update(ctx context.Context, id AgentID, updates ...AgentUpdateOptionFunc) (*Agent, error)
Query(ctx context.Context, opts ...AgentQueryOptionFunc) ([]*Agent, int, error)
@ -19,8 +23,9 @@ type AgentQueryOptionFunc func(*AgentQueryOptions)
type AgentQueryOptions struct {
Limit *int
Offset *int
RemoteIDs []string
IDs []AgentID
Thumbprints []string
Metadata *map[string]any
Statuses []AgentStatus
}
@ -36,9 +41,9 @@ func WithAgentQueryOffset(offset int) AgentQueryOptionFunc {
}
}
func WithAgentQueryRemoteID(remoteIDs ...string) AgentQueryOptionFunc {
func WithAgentQueryMetadata(metadata map[string]any) AgentQueryOptionFunc {
return func(opts *AgentQueryOptions) {
opts.RemoteIDs = remoteIDs
opts.Metadata = &metadata
}
}
@ -54,10 +59,19 @@ func WithAgentQueryStatus(statuses ...AgentStatus) AgentQueryOptionFunc {
}
}
func WithAgentQueryThumbprints(thumbprints ...string) AgentQueryOptionFunc {
return func(opts *AgentQueryOptions) {
opts.Thumbprints = thumbprints
}
}
type AgentUpdateOptionFunc func(*AgentUpdateOptions)
type AgentUpdateOptions struct {
Status *AgentStatus
Metadata *map[string]any
KeySet *jwk.Set
Thumbprint *string
}
func WithAgentUpdateStatus(status AgentStatus) AgentUpdateOptionFunc {
@ -65,3 +79,21 @@ func WithAgentUpdateStatus(status AgentStatus) AgentUpdateOptionFunc {
opts.Status = &status
}
}
func WithAgentUpdateMetadata(metadata map[string]any) AgentUpdateOptionFunc {
return func(opts *AgentUpdateOptions) {
opts.Metadata = &metadata
}
}
func WithAgentUpdateKeySet(keySet jwk.Set) AgentUpdateOptionFunc {
return func(opts *AgentUpdateOptions) {
opts.KeySet = &keySet
}
}
func WithAgentUpdateThumbprint(thumbprint string) AgentUpdateOptionFunc {
return func(opts *AgentUpdateOptions) {
opts.Thumbprint = &thumbprint
}
}

118
internal/datastore/sqlite/agent_repository.go
View File

@ -3,10 +3,13 @@ package sqlite
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"time"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
@ -116,7 +119,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
count := 0
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT id, remote_id, status, created_at, updated_at FROM agents`
query := `SELECT id, thumbprint, status, created_at, updated_at FROM agents`
limit := 10
if options.Limit != nil {
@ -133,20 +136,18 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
args := []any{offset, limit}
if options.IDs != nil && len(options.IDs) > 0 {
filters += "id in ("
filter, newArgs, newParamIndex := inFilter("id", paramIndex, options.RemoteIDs)
filter, newArgs, newParamIndex := inFilter("id", paramIndex, options.IDs)
filters += filter
paramIndex = newParamIndex
args = append(args, newArgs...)
}
if options.RemoteIDs != nil && len(options.RemoteIDs) > 0 {
if options.Thumbprints != nil && len(options.Thumbprints) > 0 {
if filters != "" {
filters += " AND "
}
filter, newArgs, newParamIndex := inFilter("remote_id", paramIndex, options.RemoteIDs)
filter, newArgs, newParamIndex := inFilter("thumbprint", paramIndex, options.Thumbprints)
filters += filter
paramIndex = newParamIndex
args = append(args, newArgs...)
@ -180,10 +181,14 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
for rows.Next() {
agent := &datastore.Agent{}
if err := rows.Scan(&agent.ID, &agent.RemoteID, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
metadata := JSONMap{}
if err := rows.Scan(&agent.ID, &agent.Thumbprint, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
return errors.WithStack(err)
}
agent.Metadata = metadata
agents = append(agents, agent)
}
@ -202,12 +207,12 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
}
// Create implements datastore.AgentRepository
func (r *AgentRepository) Create(ctx context.Context, remoteID string, status datastore.AgentStatus) (*datastore.Agent, error) {
func (r *AgentRepository) Create(ctx context.Context, thumbprint string, keySet jwk.Set, metadata map[string]any) (*datastore.Agent, error) {
agent := &datastore.Agent{}
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT count(id) FROM agents WHERE remote_id = $1`
row := tx.QueryRowContext(ctx, query, remoteID)
query := `SELECT count(id) FROM agents WHERE thumbprint = $1`
row := tx.QueryRowContext(ctx, query, thumbprint)
var count int
@ -222,21 +227,37 @@ func (r *AgentRepository) Create(ctx context.Context, remoteID string, status da
now := time.Now().UTC()
query = `
INSERT INTO agents (remote_id, status, created_at, updated_at)
VALUES($1, $2, $3, $3)
RETURNING "id", "remote_id", "status", "created_at", "updated_at"
INSERT INTO agents (thumbprint, keyset, metadata, status, created_at, updated_at)
VALUES($1, $2, $3, $4, $5, $5)
RETURNING "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
`
row = tx.QueryRowContext(
ctx, query,
remoteID, status, now,
)
err := row.Scan(&agent.ID, &agent.RemoteID, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt)
rawKeySet, err := json.Marshal(keySet)
if err != nil {
return errors.WithStack(err)
}
row = tx.QueryRowContext(
ctx, query,
thumbprint, rawKeySet, JSONMap(metadata), datastore.AgentStatusPending, now,
)
metadata := JSONMap{}
err = row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt)
if err != nil {
return errors.WithStack(err)
}
agent.Metadata = metadata
keySet, err = jwk.Parse(rawKeySet)
if err != nil {
return errors.WithStack(err)
}
agent.KeySet = &datastore.SerializableKeySet{keySet}
return nil
})
if err != nil {
@ -266,14 +287,17 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
err := r.withTx(ctx, func(tx *sql.Tx) error {
query := `
SELECT "remote_id", "status", "created_at", "updated_at"
SELECT "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
FROM agents
WHERE id = $1
`
row := r.db.QueryRowContext(ctx, query, id)
if err := row.Scan(&agent.RemoteID, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
metadata := JSONMap{}
var rawKeySet []byte
if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return datastore.ErrNotFound
}
@ -281,6 +305,15 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
return errors.WithStack(err)
}
agent.Metadata = metadata
keySet := jwk.NewSet()
if err := json.Unmarshal(rawKeySet, &keySet); err != nil {
return errors.WithStack(err)
}
agent.KeySet = &datastore.SerializableKeySet{keySet}
return nil
})
if err != nil {
@ -313,23 +346,60 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
if options.Status != nil {
query += fmt.Sprintf(`, status = $%d`, index)
args = append(args, *options.Status)
index++
}
if options.KeySet != nil {
rawKeySet, err := json.Marshal(*options.KeySet)
if err != nil {
return errors.WithStack(err)
}
query += fmt.Sprintf(`, keyset = $%d`, index)
args = append(args, rawKeySet)
index++
}
if options.Thumbprint != nil {
query += fmt.Sprintf(`, thumbprint = $%d`, index)
args = append(args, *options.Thumbprint)
index++
}
if options.Metadata != nil {
query += fmt.Sprintf(`, metadata = $%d`, index)
args = append(args, JSONMap(*options.Metadata))
index++
}
query += `
WHERE id = $1
RETURNING "id","remote_id","status","updated_at","created_at"
RETURNING "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
`
row := tx.QueryRowContext(ctx, query, args...)
if err := row.Scan(&agent.ID, &agent.RemoteID, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
metadata := JSONMap{}
var rawKeySet []byte
if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return datastore.ErrNotFound
}
return errors.WithStack(err)
}
agent.Metadata = metadata
keySet := jwk.NewSet()
if err := json.Unmarshal(rawKeySet, &keySet); err != nil {
return errors.WithStack(err)
}
agent.KeySet = &datastore.SerializableKeySet{keySet}
return nil
})
if err != nil {

133
internal/jwk/jwk.go Normal file
View File

@ -0,0 +1,133 @@
package jwk
import (
"crypto/rand"
"crypto/rsa"
"encoding/json"
"io/ioutil"
"os"
"github.com/btcsuite/btcd/btcutil/base58"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/lestrrat-go/jwx/v2/jws"
"github.com/pkg/errors"
)
const DefaultKeySize = 2048
type (
Key = jwk.Key
Set = jwk.Set
ParseOption = jwk.ParseOption
)
func Parse(src []byte, options ...jwk.ParseOption) (Set, error) {
return jwk.Parse(src, options...)
}
func PublicKeySet(keys ...jwk.Key) (jwk.Set, error) {
set := jwk.NewSet()
for _, k := range keys {
pubkey, err := k.PublicKey()
if err != nil {
return nil, errors.WithStack(err)
}
if err := pubkey.Set(jwk.AlgorithmKey, jwa.RS256); err != nil {
return nil, errors.WithStack(err)
}
if err := set.AddKey(pubkey); err != nil {
return nil, errors.WithStack(err)
}
}
return set, nil
}
func LoadOrGenerate(path string, size int) (jwk.Key, error) {
data, err := ioutil.ReadFile(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
if errors.Is(err, os.ErrNotExist) {
key, err := Generate(size)
if err != nil {
return nil, errors.WithStack(err)
}
data, err = json.Marshal(key)
if err != nil {
return nil, errors.WithStack(err)
}
if err := ioutil.WriteFile(path, data, 0o640); err != nil {
return nil, errors.WithStack(err)
}
}
key, err := jwk.ParseKey(data)
if err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}
func Generate(size int) (jwk.Key, error) {
privKey, err := rsa.GenerateKey(rand.Reader, size)
if err != nil {
return nil, errors.WithStack(err)
}
key, err := jwk.FromRaw(privKey)
if err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}
func Sign(key jwk.Key, payload ...any) (string, error) {
json, err := json.Marshal(payload)
if err != nil {
return "", errors.WithStack(err)
}
rawSignature, err := jws.Sign(
nil,
jws.WithKey(jwa.RS256, key),
jws.WithDetachedPayload(json),
)
if err != nil {
return "", errors.WithStack(err)
}
signature := base58.Encode(rawSignature)
return signature, nil
}
func Verify(jwks jwk.Set, signature string, payload ...any) (bool, error) {
json, err := json.Marshal(payload)
if err != nil {
return false, errors.WithStack(err)
}
decoded := base58.Decode(signature)
_, err = jws.Verify(
decoded,
jws.WithKeySet(jwks, jws.WithRequireKid(false)),
jws.WithDetachedPayload(json),
)
if err != nil {
return false, errors.WithStack(err)
}
return true, nil
}

40
internal/jwk/jwk_test.go Normal file
View File

@ -0,0 +1,40 @@
package jwk
import (
"testing"
"github.com/pkg/errors"
)
func TestJWK(t *testing.T) {
privateKey, err := Generate(DefaultKeySize)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
keySet, err := PublicKeySet(privateKey)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
metadata := map[string]any{
"Foo": "bar",
"Test": 1,
}
signature, err := Sign(privateKey, metadata)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
t.Logf("Signature: %s", signature)
matches, err := Verify(keySet, signature, metadata)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if !matches {
t.Error("signature should match")
}
}

0
internal/agent/machineid/get.go → internal/machineid/get.go
View File

88
internal/server/agent_api.go
View File

@ -1,11 +1,14 @@
package server
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
"forge.cadoles.com/Cadoles/emissary/internal/datastore"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"github.com/go-chi/chi"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/api"
@ -15,11 +18,14 @@ import (
const (
ErrCodeUnknownError api.ErrorCode = "unknown-error"
ErrCodeNotFound api.ErrorCode = "not-found"
ErrCodeAlreadyRegistered api.ErrorCode = "already-registered"
ErrInvalidSignature api.ErrorCode = "invalid-signature"
)
type registerAgentRequest struct {
RemoteID string `json:"remoteId"`
KeySet json.RawMessage `json:"keySet" validate:"required"`
Metadata []metadata.Tuple `json:"metadata" validate:"required"`
Thumbprint string `json:"thumbprint" validate:"required"`
Signature string `json:"signature" validate:"required"`
}
func (s *Server) registerAgent(w http.ResponseWriter, r *http.Request) {
@ -30,25 +36,81 @@ func (s *Server) registerAgent(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
agent, err := s.agentRepo.Create(
ctx,
registerAgentReq.RemoteID,
datastore.AgentStatusPending,
)
keySet, err := jwk.Parse(registerAgentReq.KeySet)
if err != nil {
if errors.Is(err, datastore.ErrAlreadyExist) {
logger.Error(ctx, "agent already registered", logger.F("remoteID", registerAgentReq.RemoteID))
api.ErrorResponse(w, http.StatusConflict, ErrCodeAlreadyRegistered, nil)
logger.Error(ctx, "could not parse key set", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil)
return
}
ctx = logger.With(ctx, logger.F("agentThumbprint", registerAgentReq.Thumbprint))
validSignature, err := jwk.Verify(keySet, registerAgentReq.Signature, registerAgentReq.Thumbprint, registerAgentReq.Metadata)
if err != nil {
logger.Error(ctx, "could not validate signature", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil)
return
}
if !validSignature {
logger.Error(ctx, "invalid signature", logger.F("signature", registerAgentReq.Signature))
api.ErrorResponse(w, http.StatusBadRequest, ErrInvalidSignature, nil)
return
}
metadata := metadata.FromSorted(registerAgentReq.Metadata)
agent, err := s.agentRepo.Create(
ctx,
registerAgentReq.Thumbprint,
keySet,
metadata,
)
if err != nil {
if !errors.Is(err, datastore.ErrAlreadyExist) {
logger.Error(ctx, "could not create agent", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil)
return
}
agents, _, err := s.agentRepo.Query(
ctx,
datastore.WithAgentQueryThumbprints(registerAgentReq.Thumbprint),
datastore.WithAgentQueryLimit(1),
)
if err != nil {
logger.Error(ctx, "could not retrieve agents", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil)
return
}
if len(agents) == 0 {
logger.Error(ctx, "could not retrieve matching agent", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeNotFound, nil)
return
}
agent, err = s.agentRepo.Update(
ctx, agents[0].ID,
datastore.WithAgentUpdateKeySet(keySet),
datastore.WithAgentUpdateMetadata(metadata),
datastore.WithAgentUpdateThumbprint(registerAgentReq.Thumbprint),
)
if err != nil {
logger.Error(ctx, "could not update agent", logger.E(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, ErrCodeUnknownError, nil)
return
}
}
api.DataResponse(w, http.StatusCreated, struct {
Agent *datastore.Agent `json:"agent"`
}{
@ -132,13 +194,13 @@ func (s *Server) queryAgents(w http.ResponseWriter, r *http.Request) {
options = append(options, datastore.WithAgentQueryID(agentIDs...))
}
remoteIDs, ok := getStringSliceValues(w, r, "remoteIds", nil)
thumbprints, ok := getStringSliceValues(w, r, "thumbprints", nil)
if !ok {
return
}
if remoteIDs != nil {
options = append(options, datastore.WithAgentQueryRemoteID(remoteIDs...))
if thumbprints != nil {
options = append(options, datastore.WithAgentQueryThumbprints(thumbprints...))
}
statuses, ok := getIntSliceValues(w, r, "statuses", nil)

2
internal/server/spec_api.go
View File

@ -33,7 +33,7 @@ func (s *Server) updateSpec(w http.ResponseWriter, r *http.Request) {
return
}
if ok, err := spec.Validate(ctx, updateSpecReq); !ok || err != nil {
if err := spec.Validate(ctx, updateSpecReq); err != nil {
data := struct {
Message string `json:"message"`
}{}

24
internal/spec/gateway/validator_test.go
View File

@ -13,24 +13,24 @@ import (
type validatorTestCase struct {
Name string
Source string
ExpectedResult bool
ShouldFail bool
}
var validatorTestCases = []validatorTestCase{
{
Name: "SpecOK",
Source: "testdata/spec-ok.json",
ExpectedResult: true,
ShouldFail: false,
},
{
Name: "SpecMissingProp",
Source: "testdata/spec-missing-prop.json",
ExpectedResult: false,
ShouldFail: true,
},
{
Name: "SpecAdditionalProp",
Source: "testdata/spec-additional-prop.json",
ExpectedResult: false,
ShouldFail: true,
},
}
@ -43,7 +43,7 @@ func TestValidator(t *testing.T) {
}
for _, tc := range validatorTestCases {
func(tc *validatorTestCase) {
func(tc validatorTestCase) {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
@ -60,16 +60,16 @@ func TestValidator(t *testing.T) {
ctx := context.Background()
result, err := validator.Validate(ctx, &spec)
err = validator.Validate(ctx, &spec)
if e, g := tc.ExpectedResult, result; e != g {
t.Errorf("result: expected '%v', got '%v'", e, g)
}
if tc.ExpectedResult && err != nil {
if !tc.ShouldFail && err != nil {
t.Errorf("+%v", errors.WithStack(err))
}
if tc.ShouldFail && err == nil {
t.Error("validation should have failed")
}
})
}(&tc)
}(tc)
}
}

8
internal/spec/spec.go
View File

@ -1,15 +1,17 @@
package spec
import "forge.cadoles.com/Cadoles/emissary/internal/agent/metadata"
type Spec interface {
SpecName() Name
SpecRevision() int
SpecData() map[string]any
SpecData() metadata.Metadata
}
type RawSpec struct {
Name Name `json:"name"`
Revision int `json:"revision"`
Data map[string]any `json:"data"`
Data metadata.Metadata `json:"data"`
}
func (s *RawSpec) SpecName() Name {
@ -20,6 +22,6 @@ func (s *RawSpec) SpecRevision() int {
return s.Revision
}
func (s *RawSpec) SpecData() map[string]any {
func (s *RawSpec) SpecData() metadata.Metadata {
return s.Data
}

22
internal/spec/uci/validator_test.go
View File

@ -13,19 +13,19 @@ import (
type validatorTestCase struct {
Name string
Source string
ExpectedResult bool
ShouldFail bool
}
var validatorTestCases = []validatorTestCase{
{
Name: "SpecOK",
Source: "testdata/spec-ok.json",
ExpectedResult: true,
ShouldFail: false,
},
{
Name: "SpecMissingProp",
Source: "testdata/spec-missing-prop.json",
ExpectedResult: false,
ShouldFail: true,
},
}
@ -38,7 +38,7 @@ func TestValidator(t *testing.T) {
}
for _, tc := range validatorTestCases {
func(tc *validatorTestCase) {
func(tc validatorTestCase) {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
@ -55,16 +55,16 @@ func TestValidator(t *testing.T) {
ctx := context.Background()
result, err := validator.Validate(ctx, &spec)
err = validator.Validate(ctx, &spec)
if e, g := tc.ExpectedResult, result; e != g {
t.Errorf("result: expected '%v', got '%v'", e, g)
}
if tc.ExpectedResult && err != nil {
if !tc.ShouldFail && err != nil {
t.Errorf("+%v", errors.WithStack(err))
}
if tc.ShouldFail && err == nil {
t.Error("validation should have failed")
}
})
}(&tc)
}(tc)
}
}

19
internal/spec/validator.go
View File

@ -23,18 +23,18 @@ func (v *Validator) Register(name Name, rawSchema []byte) error {
return nil
}
func (v *Validator) Validate(ctx context.Context, spec Spec) (bool, error) {
func (v *Validator) Validate(ctx context.Context, spec Spec) error {
schema, exists := v.schemas[spec.SpecName()]
if !exists {
return false, errors.WithStack(ErrUnknownSchema)
return errors.WithStack(ErrUnknownSchema)
}
state := schema.Validate(ctx, spec.SpecData())
state := schema.Validate(ctx, map[string]any(spec.SpecData()))
if !state.IsValid() {
return false, errors.WithStack(&ValidationError{*state.Errs})
return errors.WithStack(&ValidationError{*state.Errs})
}
return true, nil
return nil
}
func NewValidator() *Validator {
@ -53,11 +53,10 @@ func Register(name Name, rawSchema []byte) error {
return nil
}
func Validate(ctx context.Context, spec Spec) (bool, error) {
ok, err := defaultValidator.Validate(ctx, spec)
if err != nil {
return ok, errors.WithStack(err)
func Validate(ctx context.Context, spec Spec) error {
if err := defaultValidator.Validate(ctx, spec); err != nil {
return errors.WithStack(err)
}
return ok, nil
return nil
}

4
migrations/sqlite/0000000_init.up.sql
View File

@ -2,7 +2,9 @@
CREATE TABLE agents (
id INTEGER PRIMARY KEY,
remote_id TEXT NOT NULL UNIQUE,
thumbprint TEXT UNIQUE,
keyset TEXT,
metadata TEXT,
status INTEGER NOT NULL,
created_at datetime NOT NULL,
updated_at datetime NOT NULL

3
modd.conf
View File

@ -6,6 +6,7 @@ tmp/config.yml
prep: make build-emissary
prep: make tmp/server.yml
prep: make tmp/agent.yml
prep: make run-emissary-server EMISSARY_CMD="--debug --config tmp/agent.yml server database migrate"
daemon: make run-emissary-server EMISSARY_CMD="--debug --config tmp/server.yml server run"
# daemon: make run-emissary-agent EMISSARY_CMD="--debug --config tmp/agent.yml agent run"
daemon: make run-emissary-agent EMISSARY_CMD="--debug --config tmp/agent.yml agent run"
}