feat(storage,agent): add label attribute

.gitignore

@@ -4,7 +4,7 @@ dist/
 /tools
 /tmp
 /state.json
-/emissary.sqlite
+/emissary.sqlite*
 /.gitea-release
 /agent-key.json
 /apps

Jenkinsfile

@@ -0,0 +1,70 @@
+@Library('cadoles') _
+
+pipeline {
+    agent {
+        dockerfile {
+            label 'docker'
+            filename 'Dockerfile'
+            dir 'misc/jenkins'
+        }
+    }
+
+    stages {
+        stage('Run unit tests') {
+            steps {
+                script {
+                    withCredentials([
+                        usernamePassword([
+                        credentialsId: 'forge-jenkins',
+                        usernameVariable: 'GIT_USERNAME',
+                        passwordVariable: 'GIT_PASSWORD'
+                        ])
+                    ]) {
+                        sh '''
+                        git config --global credential.https://forge.cadoles.com.username "$GIT_USERNAME"
+                        git config --global credential.https://forge.cadoles.com.helper '!f() { test "$1" = get && echo "password=$GIT_PASSWORD"; }; f'
+
+                        export GOPRIVATE=forge.cadoles.com/arcad/edge
+                        make test
+                        '''
+                    }
+                }
+            }
+        }
+
+        stage('Release') {
+            when {
+                anyOf {
+                    branch 'master'
+                    branch 'develop'
+                }
+            }
+            steps {
+                script {
+                    withCredentials([
+                        usernamePassword([
+                        credentialsId: 'forge-jenkins',
+                        usernameVariable: 'GITEA_RELEASE_USERNAME',
+                        passwordVariable: 'GITEA_RELEASE_PASSWORD'
+                        ])
+                    ]) {
+                        sh 'make gitea-release'
+                    }
+                    def currentVersion = sh(returnStdout: true, script: 'make full-version').trim()
+                    build(
+                        job: "../emissary-firmware/${env.GIT_BRANCH}",
+                        parameters: [
+                            [$class: 'StringParameterValue', name: 'emissaryRelease', value: currentVersion]
+                        ]
+                    )
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            cleanWs()
+        }
+    }
+}

Makefile

@@ -135,7 +135,7 @@ gitea-release: tools/gitea-release/bin/gitea-release.sh goreleaser
 		GITEA_RELEASE_COMMITISH_TARGET="$(GIT_VERSION)" \
 		GITEA_RELEASE_IS_DRAFT="false" \
 		GITEA_RELEASE_BODY="" \
-		GITEA_RELEASE_ATTACHMENTS="$(shell find .gitea-release/* -type f)" \
+		GITEA_RELEASE_ATTACHMENTS="$$(find .gitea-release/* -type f)" \
 		tools/gitea-release/bin/gitea-release.sh
 
 tools/gitea-release/bin/gitea-release.sh:
@@ -150,4 +150,7 @@ AGENT_ID ?= 1
 
 load-sample-specs:
 	cat misc/spec-samples/app.emissary.cadoles.com.json | ./bin/server api agent spec update -a $(AGENT_ID) --no-patch --spec-data - --spec-name app.emissary.cadoles.com
-	cat misc/spec-samples/proxy.emissary.cadoles.com.json | ./bin/server api agent spec update -a $(AGENT_ID) --no-patch --spec-data - --spec-name proxy.emissary.cadoles.com
+	cat misc/spec-samples/proxy.emissary.cadoles.com.json | ./bin/server api agent spec update -a $(AGENT_ID) --no-patch --spec-data - --spec-name proxy.emissary.cadoles.com
+
+full-version:
+	@echo $(FULL_VERSION)

cmd/agent/main.go

@@ -7,6 +7,7 @@ import (
 	"forge.cadoles.com/Cadoles/emissary/internal/command/agent"
 	"forge.cadoles.com/Cadoles/emissary/internal/command/api"
 
+	_ "forge.cadoles.com/Cadoles/emissary/internal/imports/format"
 	_ "forge.cadoles.com/Cadoles/emissary/internal/imports/spec"
 )
 

internal/agent/controller/app/app_handler.go

@@ -28,6 +28,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+const defaultSQLiteParams = "?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate"
+
 func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs *spec.Spec) ([]edgeHTTP.HandlerOptionFunc, error) {
 	dataDir, err := c.ensureAppDataDir(ctx, appKey)
 	if err != nil {
@@ -35,7 +37,7 @@ func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs
 	}
 
 	dbFile := filepath.Join(dataDir, appKey+".sqlite")
-	db, err := sqlite.Open(dbFile)
+	db, err := sqlite.Open(dbFile + defaultSQLiteParams)
 	if err != nil {
 		return nil, errors.Wrapf(err, "could not open database file '%s'", dbFile)
 	}

internal/agent/controller/app/controller.go

@@ -16,8 +16,8 @@ import (
 )
 
 type serverEntry struct {
-	SpecHash uint64
+	AppDefHash uint64
-	Server   *Server
+	Server     *Server
 }
 
 type Controller struct {
@@ -124,7 +124,20 @@ func (c *Controller) updateAppRepository(ctx context.Context, specs *spec.Spec)
 func (c *Controller) updateApp(ctx context.Context, specs *spec.Spec, appKey string) (err error) {
 	appEntry := specs.Apps[appKey]
 
-	newAppSpecHash, err := hashstructure.Hash(appEntry, hashstructure.FormatV2, nil)
+	var auth *spec.Auth
+	if specs.Config != nil {
+		auth = specs.Config.Auth
+	}
+
+	appDef := struct {
+		App  spec.AppEntry
+		Auth *spec.Auth
+	}{
+		App:  appEntry,
+		Auth: auth,
+	}
+
+	newAppDefHash, err := hashstructure.Hash(appDef, hashstructure.FormatV2, nil)
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -163,20 +176,20 @@ func (c *Controller) updateApp(ctx context.Context, specs *spec.Spec, appKey str
 		}
 
 		server = &serverEntry{
-			Server:   NewServer(bundle, auth, options...),
+			Server:     NewServer(bundle, auth, options...),
-			SpecHash: 0,
+			AppDefHash: 0,
 		}
 
 		c.servers[appKey] = server
 	}
 
-	specChanged := newAppSpecHash != server.SpecHash
+	defChanged := newAppDefHash != server.AppDefHash
 
-	if server.Server.Running() && !specChanged {
+	if server.Server.Running() && !defChanged {
 		return nil
 	}
 
-	if specChanged && server.SpecHash != 0 {
+	if defChanged && server.AppDefHash != 0 {
 		logger.Info(
 			ctx, "restarting app",
 			logger.F("address", appEntry.Address),
@@ -194,7 +207,7 @@ func (c *Controller) updateApp(ctx context.Context, specs *spec.Spec, appKey str
 		return errors.Wrap(err, "could not start app")
 	}
 
-	server.SpecHash = newAppSpecHash
+	server.AppDefHash = newAppDefHash
 
 	return nil
 }

internal/agent/controller/app/server.go

@@ -3,11 +3,13 @@ package app
 import (
 	"context"
 	"net/http"
+	"strings"
 	"sync"
 	"time"
 
 	"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
 	appSpec "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
+	"forge.cadoles.com/Cadoles/emissary/internal/proxy/wildcard"
 	edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
 	authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
 	"gitlab.com/wpetit/goweb/logger"
@@ -33,12 +35,15 @@ type Server struct {
 }
 
 func (s *Server) Start(ctx context.Context, addr string) (err error) {
-	if s.server != nil {
+	if s.Running() {
 		if err := s.Stop(); err != nil {
 			return errors.WithStack(err)
 		}
 	}
 
+	s.serverMutex.Lock()
+	defer s.serverMutex.Unlock()
+
 	router := chi.NewRouter()
 
 	router.Use(middleware.Logger)
@@ -83,9 +88,7 @@ func (s *Server) Start(ctx context.Context, addr string) (err error) {
 		}
 	}()
 
-	s.serverMutex.Lock()
 	s.server = server
-	s.serverMutex.Unlock()
 
 	return nil
 }
@@ -98,20 +101,25 @@ func (s *Server) Running() bool {
 }
 
 func (s *Server) Stop() error {
+	if !s.Running() {
+		return nil
+	}
+
+	s.serverMutex.Lock()
+	defer s.serverMutex.Unlock()
+
 	if s.server == nil {
 		return nil
 	}
 
-	defer func() {
-		s.serverMutex.Lock()
-		s.server = nil
-		s.serverMutex.Unlock()
-	}()
-
 	if err := s.server.Close(); err != nil {
-		panic(errors.WithStack(err))
+		s.server = nil
+
+		return errors.WithStack(err)
 	}
 
+	s.server = nil
+
 	return nil
 }
 
@@ -140,6 +148,10 @@ func (s *Server) configureAuth(router chi.Router, auth *spec.Auth) error {
 			}
 		}
 
+		if s.auth.Local.CookieDomain != "" {
+			router.Use(invalidCookieDomainRedirect(s.auth.Local.CookieDomain))
+		}
+
 		router.Handle("/auth/*", authHTTP.NewLocalHandler(
 			jwa.HS256, key,
 			authHTTP.WithRoutePrefix("/auth"),
@@ -158,3 +170,33 @@ func NewServer(bundle bundle.Bundle, auth *appSpec.Auth, handlerOptions ...edgeH
 		handlerOptions: handlerOptions,
 	}
 }
+
+func invalidCookieDomainRedirect(cookieDomain string) func(http.Handler) http.Handler {
+	domain := strings.TrimPrefix(cookieDomain, ".")
+	hostPattern := "*" + domain
+
+	return func(h http.Handler) http.Handler {
+		fn := func(w http.ResponseWriter, r *http.Request) {
+			hostParts := strings.SplitN(r.Host, ":", 2)
+
+			if !wildcard.Match(hostParts[0], hostPattern) {
+				url := r.URL
+
+				newHost := domain
+				if len(hostParts) > 1 {
+					newHost += ":" + hostParts[1]
+				}
+
+				url.Host = newHost
+
+				http.Redirect(w, r, url.String(), http.StatusTemporaryRedirect)
+
+				return
+			}
+
+			h.ServeHTTP(w, r)
+		}
+
+		return http.HandlerFunc(fn)
+	}
+}

internal/auth/agent/jwt.go

@@ -18,7 +18,7 @@ func GenerateToken(key jwk.Key, thumbprint string) (string, error) {
 		return "", errors.WithStack(err)
 	}
 
-	now := time.Now()
+	now := time.Now().UTC()
 
 	if err := token.Set(jwt.NotBeforeKey, now); err != nil {
 		return "", errors.WithStack(err)

internal/auth/thirdparty/jwt.go

@@ -42,7 +42,7 @@ func GenerateToken(ctx context.Context, key jwk.Key, issuer, subject string, rol
 		return "", errors.WithStack(err)
 	}
 
-	now := time.Now()
+	now := time.Now().UTC()
 
 	if err := token.Set(jwt.NotBeforeKey, now); err != nil {
 		return "", errors.WithStack(err)

internal/client/update_agent.go

@@ -10,6 +10,7 @@ import (
 
 type UpdateAgentOptions struct {
 	Status  *int
+	Label   *string
 	Options []OptionFunc
 }
 
@@ -21,6 +22,12 @@ func WithAgentStatus(status int) UpdateAgentOptionFunc {
 	}
 }
 
+func WithAgentLabel(label string) UpdateAgentOptionFunc {
+	return func(opts *UpdateAgentOptions) {
+		opts.Label = &label
+	}
+}
+
 func WithUpdateAgentsOptions(funcs ...OptionFunc) UpdateAgentOptionFunc {
 	return func(opts *UpdateAgentOptions) {
 		opts.Options = funcs
@@ -39,6 +46,10 @@ func (c *Client) UpdateAgent(ctx context.Context, agentID datastore.AgentID, fun
 		payload["status"] = *opts.Status
 	}
 
+	if opts.Label != nil {
+		payload["label"] = *opts.Label
+	}
+
 	response := withResponse[struct {
 		Agent *datastore.Agent `json:"agent"`
 	}]()

internal/command/agent/run.go

@@ -49,10 +49,6 @@ func RunCommand() *cli.Command {
 				controllers = append(controllers, spec.NewController())
 			}
 
-			if ctrlConf.Proxy.Enabled {
-				controllers = append(controllers, proxy.NewController())
-			}
-
 			if ctrlConf.UCI.Enabled {
 				controllers = append(controllers, openwrt.NewUCIController(
 					string(ctrlConf.UCI.BinPath),
@@ -66,6 +62,10 @@ func RunCommand() *cli.Command {
 				))
 			}
 
+			if ctrlConf.Proxy.Enabled {
+				controllers = append(controllers, proxy.NewController())
+			}
+
 			if ctrlConf.SysUpgrade.Enabled {
 				sysUpgradeArgs := make([]string, 0)
 				if len(ctrlConf.SysUpgrade.SysUpgradeCommand) > 1 {

internal/command/api/agent/update.go

@@ -22,6 +22,11 @@ func UpdateCommand() *cli.Command {
 				Usage: "Set `STATUS` to selected agent",
 				Value: -1,
 			},
+			&cli.StringFlag{
+				Name:  "label",
+				Usage: "Set `LABEL` to selected agent",
+				Value: "",
+			},
 		),
 		Action: func(ctx *cli.Context) error {
 			baseFlags := clientFlag.GetBaseFlags(ctx)
@@ -43,6 +48,11 @@ func UpdateCommand() *cli.Command {
 				options = append(options, client.WithAgentStatus(status))
 			}
 
+			label := ctx.String("label")
+			if label != "" {
+				options = append(options, client.WithAgentLabel(label))
+			}
+
 			client := client.New(baseFlags.ServerURL, client.WithToken(token))
 
 			agent, err := client.UpdateAgent(ctx.Context, agentID, options...)

internal/command/api/agent/util.go

@@ -7,6 +7,7 @@ func agentHints(outputMode format.OutputMode) format.Hints {
 		OutputMode: outputMode,
 		Props: []format.Prop{
 			format.NewProp("ID", "ID"),
+			format.NewProp("Label", "Label"),
 			format.NewProp("Thumbprint", "Thumbprint"),
 			format.NewProp("Status", "Status"),
 			format.NewProp("CreatedAt", "CreatedAt"),

internal/config/database.go

@@ -15,6 +15,6 @@ type DatabaseConfig struct {
 func NewDefaultDatabaseConfig() DatabaseConfig {
 	return DatabaseConfig{
 		Driver: "sqlite",
-		DSN:    "sqlite://emissary.sqlite",
+		DSN:    "sqlite://emissary.sqlite?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate",
 	}
 }

internal/datastore/agent.go

@@ -21,6 +21,7 @@ const (
 
 type Agent struct {
 	ID         AgentID             `json:"id"`
+	Label      string              `json:"label"`
 	Thumbprint string              `json:"thumbprint"`
 	KeySet     *SerializableKeySet `json:"keyset,omitempty"`
 	Metadata   map[string]any      `json:"metadata,omitempty"`

internal/datastore/agent_repository.go

@@ -68,6 +68,7 @@ func WithAgentQueryThumbprints(thumbprints ...string) AgentQueryOptionFunc {
 type AgentUpdateOptionFunc func(*AgentUpdateOptions)
 
 type AgentUpdateOptions struct {
+	Label      *string
 	Status     *AgentStatus
 	Metadata   *map[string]any
 	KeySet     *jwk.Set
@@ -97,3 +98,9 @@ func WithAgentUpdateThumbprint(thumbprint string) AgentUpdateOptionFunc {
 		opts.Thumbprint = &thumbprint
 	}
 }
+
+func WithAgentUpdateLabel(label string) AgentUpdateOptionFunc {
+	return func(opts *AgentUpdateOptions) {
+		opts.Label = &label
+	}
+}

internal/datastore/sqlite/agent_repository.go

@@ -45,7 +45,11 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI
 		return nil, errors.WithStack(err)
 	}
 
-	defer rows.Close()
+	defer func() {
+		if err := rows.Close(); err != nil {
+			logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
+		}
+	}()
 
 	for rows.Next() {
 		spec := &datastore.Spec{}
@@ -61,6 +65,10 @@ func (r *AgentRepository) GetSpecs(ctx context.Context, agentID datastore.AgentI
 		specs = append(specs, spec)
 	}
 
+	if err := rows.Err(); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	return specs, nil
 }
 
@@ -119,7 +127,7 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 	count := 0
 
 	err := r.withTx(ctx, func(tx *sql.Tx) error {
-		query := `SELECT id, thumbprint, status, created_at, updated_at FROM agents`
+		query := `SELECT id, label, thumbprint, status, created_at, updated_at FROM agents`
 
 		limit := 10
 		if options.Limit != nil {
@@ -176,14 +184,18 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 			return errors.WithStack(err)
 		}
 
-		defer rows.Close()
+		defer func() {
+			if err := rows.Close(); err != nil {
+				logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
+			}
+		}()
 
 		for rows.Next() {
 			agent := &datastore.Agent{}
 
 			metadata := JSONMap{}
 
-			if err := rows.Scan(&agent.ID, &agent.Thumbprint, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+			if err := rows.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 				return errors.WithStack(err)
 			}
 
@@ -192,6 +204,10 @@ func (r *AgentRepository) Query(ctx context.Context, opts ...datastore.AgentQuer
 			agents = append(agents, agent)
 		}
 
+		if err := rows.Err(); err != nil {
+			return errors.WithStack(err)
+		}
+
 		row := tx.QueryRowContext(ctx, `SELECT count(id) FROM agents `+filters, args...)
 		if err := row.Scan(&count); err != nil {
 			return errors.WithStack(err)
@@ -299,7 +315,7 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 
 	err := r.withTx(ctx, func(tx *sql.Tx) error {
 		query := `
-		SELECT "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
+		SELECT "id", "label", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
 		FROM agents
 		WHERE id = $1
 		`
@@ -309,7 +325,7 @@ func (r *AgentRepository) Get(ctx context.Context, id datastore.AgentID) (*datas
 		metadata := JSONMap{}
 		var rawKeySet []byte
 
-		if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return datastore.ErrNotFound
 			}
@@ -379,6 +395,12 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 			index++
 		}
 
+		if options.Label != nil {
+			query += fmt.Sprintf(`, label = $%d`, index)
+			args = append(args, *options.Label)
+			index++
+		}
+
 		if options.Metadata != nil {
 			query += fmt.Sprintf(`, metadata = $%d`, index)
 			args = append(args, JSONMap(*options.Metadata))
@@ -387,7 +409,7 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 
 		query += `
 			WHERE id = $1 
-			RETURNING "id", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
+			RETURNING "id", "label", "thumbprint", "keyset", "metadata", "status", "created_at", "updated_at"
 		`
 
 		row := tx.QueryRowContext(ctx, query, args...)
@@ -395,7 +417,7 @@ func (r *AgentRepository) Update(ctx context.Context, id datastore.AgentID, opts
 		metadata := JSONMap{}
 		var rawKeySet []byte
 
-		if err := row.Scan(&agent.ID, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
+		if err := row.Scan(&agent.ID, &agent.Label, &agent.Thumbprint, &rawKeySet, &metadata, &agent.Status, &agent.CreatedAt, &agent.UpdatedAt); err != nil {
 			if errors.Is(err, sql.ErrNoRows) {
 				return datastore.ErrNotFound
 			}

internal/server/agent_api.go

@@ -145,6 +145,7 @@ func (s *Server) registerAgent(w http.ResponseWriter, r *http.Request) {
 
 type updateAgentRequest struct {
 	Status *datastore.AgentStatus `json:"status" validate:"omitempty,oneof=0 1 2 3"`
+	Label  *string                `json:"label" validate:"omitempty"`
 }
 
 func (s *Server) updateAgent(w http.ResponseWriter, r *http.Request) {
@@ -166,6 +167,10 @@ func (s *Server) updateAgent(w http.ResponseWriter, r *http.Request) {
 		options = append(options, datastore.WithAgentUpdateStatus(*updateAgentReq.Status))
 	}
 
+	if updateAgentReq.Label != nil {
+		options = append(options, datastore.WithAgentUpdateLabel(*updateAgentReq.Label))
+	}
+
 	agent, err := s.agentRepo.Update(
 		ctx,
 		datastore.AgentID(agentID),

migrations/sqlite/0000001_agent_label.down.sql

@@ -0,0 +1 @@
+ALTER TABLE agents DROP COLUMN label;

migrations/sqlite/0000001_agent_label.up.sql

@@ -0,0 +1 @@
+ALTER TABLE agents ADD COLUMN label TEXT DEFAULT "";

misc/jenkins/Dockerfile

@@ -0,0 +1,24 @@
+FROM reg.cadoles.com/proxy_cache/library/ubuntu:22.04
+
+ARG HTTP_PROXY=
+ARG HTTPS_PROXY=
+ARG http_proxy=
+ARG https_proxy=
+ARG GO_VERSION=1.19.2
+
+# Install dev environment dependencies
+RUN export DEBIAN_FRONTEND=noninteractive &&\
+    apt-get update -y &&\
+    apt-get install -y --no-install-recommends curl ca-certificates build-essential wget unzip tar git jq
+
+# Install Go
+RUN mkdir -p /tmp \
+    && wget -O /tmp/go${GO_VERSION}.linux-amd64.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \
+    && rm -rf /usr/local/go \
+    && mkdir -p /usr/local \
+    && tar -C /usr/local -xzf /tmp/go${GO_VERSION}.linux-amd64.tar.gz
+
+ENV PATH="${PATH}:/usr/local/go/bin"
+
+# Add LetsEncrypt certificates
+RUN curl -k https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/common/add-letsencrypt-ca.sh | bash

misc/packaging/common/config-server.yml

@@ -9,7 +9,7 @@ server:
     port: 3000
   database:
     driver: sqlite
-    dsn: sqlite:///var/lib/emissary/data.sqlite
+    dsn: sqlite:///var/lib/emissary/data.sqlite?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_txlock=immediate
   cors:
     allowedOrigins: []
     allowCredentials: true