feat(controller, app): add fetch module

go.mod

@@ -3,7 +3,7 @@ module forge.cadoles.com/Cadoles/emissary
 go 1.19
 
 require (
-	forge.cadoles.com/arcad/edge v0.0.0-20230328183829-d8ce2901d2ab
+	forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883

go.sum

@@ -56,6 +56,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 forge.cadoles.com/arcad/edge v0.0.0-20230328183829-d8ce2901d2ab h1:xOtzLAYOUcKd/VBx/PzL2riC0zNuQ/cxxf5r3AmEvJE=
 forge.cadoles.com/arcad/edge v0.0.0-20230328183829-d8ce2901d2ab/go.mod h1:ONd6vyQ0IM0vHi1i+bmZBRc1Fd0BoXMuDdY/+0sZefw=
+forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6 h1:MxMEBSEvwagUrFORUJ9snZekFIKkaV3OB0EplXra+LU=
+forge.cadoles.com/arcad/edge v0.0.0-20230402160147-f08f645432c6/go.mod h1:ONd6vyQ0IM0vHi1i+bmZBRc1Fd0BoXMuDdY/+0sZefw=
 gioui.org v0.0.0-20210308172011-57750fc8a0a6/go.mod h1:RSH6KIUZ0p2xy5zHDxgAM4zumjgTw83q2ge/PI+yyw8=
 github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg=
 github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k=

internal/agent/agent.go

@@ -44,8 +44,6 @@ func (a *Agent) Run(ctx context.Context) error {
 
 		if err := a.registerAgent(ctx, client, state); err != nil {
 			logger.Error(ctx, "could not register agent", logger.E(errors.WithStack(err)))
-
-			return
 		}
 
 		logger.Debug(ctx, "state before reconciliation", logger.F("state", state))
@@ -81,7 +79,7 @@ func (a *Agent) Reconcile(ctx context.Context, state *State) error {
 		)
 
 		if err := ctrl.Reconcile(ctrlCtx, state); err != nil {
-			return errors.WithStack(err)
+			logger.Error(ctx, "could not reconcile", logger.E(errors.WithStack(err)))
 		}
 	}
 

internal/agent/controller/app/app_handler.go

@@ -20,6 +20,7 @@ import (
 	"forge.cadoles.com/arcad/edge/pkg/module/auth"
 	"forge.cadoles.com/arcad/edge/pkg/module/blob"
 	"forge.cadoles.com/arcad/edge/pkg/module/cast"
+	fetchModule "forge.cadoles.com/arcad/edge/pkg/module/fetch"
 	"forge.cadoles.com/arcad/edge/pkg/module/net"
 	"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
 	"github.com/Masterminds/sprig/v3"
@@ -186,5 +187,6 @@ func (c *Controller) getAppModules(bus bus.Bus, db *sql.DB, spec *appSpec.Spec,
 			},
 		),
 		appModule.ModuleFactory(c.appRepository),
+		fetchModule.ModuleFactory(bus),
 	}
 }

internal/auth/agent/authenticator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"strings"
+	"time"
 
 	"forge.cadoles.com/Cadoles/emissary/internal/auth"
 	"forge.cadoles.com/Cadoles/emissary/internal/datastore"
@@ -13,8 +14,11 @@ import (
 	"gitlab.com/wpetit/goweb/logger"
 )
 
+const DefaultAcceptableSkew = 5 * time.Minute
+
 type Authenticator struct {
-	repo datastore.AgentRepository
+	repo           datastore.AgentRepository
+	acceptableSkew time.Duration
 }
 
 // Authenticate implements auth.Authenticator.
@@ -71,11 +75,19 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
 		[]byte(rawToken),
 		jwt.WithKeySet(agent.KeySet.Set, jws.WithRequireKid(false)),
 		jwt.WithValidate(true),
+		jwt.WithAcceptableSkew(a.acceptableSkew),
 	)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
+	contactedAt := time.Now()
+
+	agent, err = a.repo.Update(ctx, agent.ID, datastore.WithAgentUpdateContactedAt(contactedAt))
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	user := &User{
 		agent: agent,
 	}
@@ -83,9 +95,10 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
 	return user, nil
 }
 
-func NewAuthenticator(repo datastore.AgentRepository) *Authenticator {
+func NewAuthenticator(repo datastore.AgentRepository, acceptableSkew time.Duration) *Authenticator {
 	return &Authenticator{
-		repo: repo,
+		repo:           repo,
+		acceptableSkew: acceptableSkew,
 	}
 }
 

internal/auth/thirdparty/authenticator.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"net/http"
 	"strings"
+	"time"
 
 	"forge.cadoles.com/Cadoles/emissary/internal/auth"
 	"forge.cadoles.com/Cadoles/emissary/internal/jwk"
@@ -11,9 +12,12 @@ import (
 	"gitlab.com/wpetit/goweb/logger"
 )
 
+const DefaultAcceptableSkew = 5 * time.Minute
+
 type Authenticator struct {
-	keys   jwk.Set
+	keys           jwk.Set
-	issuer string
+	issuer         string
+	acceptableSkew time.Duration
 }
 
 // Authenticate implements auth.Authenticator.
@@ -30,7 +34,7 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
 		return nil, errors.WithStack(auth.ErrUnauthenticated)
 	}
 
-	token, err := parseToken(ctx, a.keys, a.issuer, rawToken)
+	token, err := parseToken(ctx, a.keys, a.issuer, rawToken, a.acceptableSkew)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -57,10 +61,11 @@ func (a *Authenticator) Authenticate(ctx context.Context, r *http.Request) (auth
 	return user, nil
 }
 
-func NewAuthenticator(keys jwk.Set, issuer string) *Authenticator {
+func NewAuthenticator(keys jwk.Set, issuer string, acceptableSkew time.Duration) *Authenticator {
 	return &Authenticator{
-		keys:   keys,
+		keys:           keys,
-		issuer: issuer,
+		issuer:         issuer,
+		acceptableSkew: acceptableSkew,
 	}
 }
 

internal/auth/thirdparty/jwt.go

@@ -13,12 +13,13 @@ import (
 
 const keyRole = "role"
 
-func parseToken(ctx context.Context, keys jwk.Set, issuer string, rawToken string) (jwt.Token, error) {
+func parseToken(ctx context.Context, keys jwk.Set, issuer string, rawToken string, acceptableSkew time.Duration) (jwt.Token, error) {
 	token, err := jwt.Parse(
 		[]byte(rawToken),
 		jwt.WithKeySet(keys, jws.WithRequireKid(false)),
 		jwt.WithIssuer(issuer),
 		jwt.WithValidate(true),
+		jwt.WithAcceptableSkew(acceptableSkew),
 	)
 	if err != nil {
 		return nil, errors.WithStack(err)

internal/command/api/agent/util.go

@@ -10,7 +10,7 @@ func agentHints(outputMode format.OutputMode) format.Hints {
 			format.NewProp("Label", "Label"),
 			format.NewProp("Thumbprint", "Thumbprint"),
 			format.NewProp("Status", "Status"),
-			format.NewProp("CreatedAt", "CreatedAt"),
+			format.NewProp("ContactedAt", "ContactedAt"),
 			format.NewProp("UpdatedAt", "UpdatedAt"),
 		},
 	}

internal/datastore/agent.go

@@ -20,14 +20,15 @@ const (
 )
 
 type Agent struct {
-	ID         AgentID             `json:"id"`
+	ID          AgentID             `json:"id"`
-	Label      string              `json:"label"`
+	Label       string              `json:"label"`
-	Thumbprint string              `json:"thumbprint"`
+	Thumbprint  string              `json:"thumbprint"`
-	KeySet     *SerializableKeySet `json:"keyset,omitempty"`
+	KeySet      *SerializableKeySet `json:"keyset,omitempty"`
-	Metadata   map[string]any      `json:"metadata,omitempty"`
+	Metadata    map[string]any      `json:"metadata,omitempty"`
-	Status     AgentStatus         `json:"status"`
+	Status      AgentStatus         `json:"status"`
-	CreatedAt  time.Time           `json:"createdAt"`
+	CreatedAt   time.Time           `json:"createdAt"`
-	UpdatedAt  time.Time           `json:"updatedAt"`
+	UpdatedAt   time.Time           `json:"updatedAt"`
+	ContactedAt *time.Time          `json:"contactedAt,omitempty"`
 }
 
 type SerializableKeySet struct {

internal/datastore/agent_repository.go

@@ -2,6 +2,7 @@ package datastore
 
 import (
 	"context"
+	"time"
 
 	"github.com/lestrrat-go/jwx/v2/jwk"
 )
@@ -68,11 +69,12 @@ func WithAgentQueryThumbprints(thumbprints ...string) AgentQueryOptionFunc {
 type AgentUpdateOptionFunc func(*AgentUpdateOptions)
 
 type AgentUpdateOptions struct {
-	Label      *string
+	Label       *string
-	Status     *AgentStatus
+	Status      *AgentStatus
-	Metadata   *map[string]any
+	ContactedAt *time.Time
-	KeySet     *jwk.Set
+	Metadata    *map[string]any
-	Thumbprint *string
+	KeySet      *jwk.Set
+	Thumbprint  *string
 }
 
 func WithAgentUpdateStatus(status AgentStatus) AgentUpdateOptionFunc {
@@ -104,3 +106,9 @@ func WithAgentUpdateLabel(label string) AgentUpdateOptionFunc {
 		opts.Label = &label
 	}
 }
+
+func WithAgentUpdateContactedAt(contactedAt time.Time) AgentUpdateOptionFunc {
+	return func(opts *AgentUpdateOptions) {
+		opts.ContactedAt = &contactedAt
+	}
+}

internal/datastore/sqlite/agent_repository.go

@@ -127,7 +127,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 	count := 0
 
 	err := r.withTx(ctx, func(tx *sql.Tx) error {
-		query := `SELECT id, label, thumbprint, status, created_at, updated_at FROM agents`
+		query := `SELECT id, label, thumbprint, status, contacted_at, created_at, updated_at FROM agents`
 
 		limit := 10
 		if options.Limit != nil {
@@ -194,12 +194,16 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 			agent := &datastore.Agent{}
 
 			metadata := JSONMap{}
+			contactedAt := sql.NullTime{}
 
-			if err := rows.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+			if err := rows.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 				return errors.WithStack(err)
 			}
 
 			agent.Metadata = metadata
+			if contactedAt.Valid {
+				agent.ContactedAt = &contactedAt.Time
+			}
 
 			agents = append(agents, agent)
 		}
@@ -315,7 +319,7 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 
 	err := r.withTx(ctx, func(tx *sql.Tx) error {
 		query := `
-		SELECT "id", "label", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
+		SELECT "id", "label", "thumbprint", "keyset", "metadata", "status", "contacted_at", "created_at", "updated_at"
 		FROM agents
 		WHERE id = $1
 		`
@@ -323,9 +327,10 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 		row := r.db.QueryRowContext(ctx, query, id)
 
 		metadata := JSONMap{}
+		contactedAt := sql.NullTime{}
 		var rawKeySet []byte
 
-		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return datastore.ErrNotFound
 			}
@@ -334,6 +339,9 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 		}
 
 		agent.Metadata = metadata
+		if contactedAt.Valid {
+			agent.ContactedAt = &contactedAt.Time
+		}
 
 		keySet := jwk.NewSet()
 		if err := json.Unmarshal(rawKeySet, &keySet); err != nil {
@@ -362,15 +370,11 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 
 	err := r.withTx(ctx, func(tx *sql.Tx) error {
 		query := `
-			UPDATE agents SET updated_at = $2
+			UPDATE agents SET id = $1
 		`
 
-		now := time.Now().UTC()
+		args := []any{id}
-
+		index := 2
-		args := []any{
-			id, now,
-		}
-		index := 3
 
 		if options.Status != nil {
 			query += fmt.Sprintf(`, status = $%d`, index)
@@ -401,23 +405,45 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 			index++
 		}
 
+		if options.ContactedAt != nil {
+			query += fmt.Sprintf(`, contacted_at = $%d`, index)
+			utc := options.ContactedAt.UTC()
+			args = append(args, utc)
+			index++
+		}
+
 		if options.Metadata != nil {
 			query += fmt.Sprintf(`, metadata = $%d`, index)
 			args = append(args, JSONMap(*options.Metadata))
 			index++
 		}
 
+		updated := options.Metadata != nil ||
+			options.Status != nil ||
+			options.Label != nil ||
+			options.KeySet != nil ||
+			options.Thumbprint != nil
+		if updated {
+			now := time.Now().UTC()
+			query += fmt.Sprintf(`, updated_at = $%d`, index)
+			args = append(args, now)
+			index++
+		}
+
 		query += `
 			WHERE id = $1 
-			RETURNING "id", "label", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
+			RETURNING "id", "label", "thumbprint", "keyset", "metadata", "status", "contacted_at", "created_at", "updated_at"
 		`
 
+		logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
+
 		row := tx.QueryRowContext(ctx, query, args...)
 
 		metadata := JSONMap{}
+		contactedAt := sql.NullTime{}
 		var rawKeySet []byte
 
-		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &contactedAt, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return datastore.ErrNotFound
 			}
@@ -426,6 +452,9 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 		}
 
 		agent.Metadata = metadata
+		if contactedAt.Valid {
+			agent.ContactedAt = &contactedAt.Time
+		}
 
 		keySet := jwk.NewSet()
 		if err := json.Unmarshal(rawKeySet, &keySet); err != nil {

internal/server/server.go

@@ -105,8 +105,8 @@ func (s *Server) run(parentCtx context.Context, addrs chan net.Addr, errs chan e
 
 		r.Group(func(r chi.Router) {
 			r.Use(auth.Middleware(
-				thirdparty.NewAuthenticator(keys, string(s.conf.Issuer)),
+				thirdparty.NewAuthenticator(keys, string(s.conf.Issuer), thirdparty.DefaultAcceptableSkew),
-				agent.NewAuthenticator(s.agentRepo),
+				agent.NewAuthenticator(s.agentRepo, agent.DefaultAcceptableSkew),
 			))
 
 			r.Route("/agents", func(r chi.Router) {

migrations/sqlite/0000002_agent_contactedat.down.sql

@@ -0,0 +1 @@
+ALTER TABLE agents DROP COLUMN contacted_at;

migrations/sqlite/0000002_agent_contactedat.up.sql

@@ -0,0 +1 @@
+ALTER TABLE agents ADD COLUMN contacted_at datetime;