arcad/emissary
16
0
Fork 0
Code Issues 14 Pull Requests Packages Projects Releases 3 Wiki Activity

feat(edge): integrate new dsn based storage system
Some checks failed
arcad/emissary/pipeline/head There was a failure building this commit
Details

Browse Source
This commit is contained in:
wpetit
2023-10-02 22:09:15 -06:00
parent 75cab3264f
commit 6c78bc5c7c
9 changed files with 2185 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
internal/agent/controller/app/app_handler.go
View File

@ -4,7 +4,6 @@ import (
"bytes"
"context"
"net"
"path/filepath"
"text/template"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
@ -20,44 +19,54 @@ import (
fetchModule "forge.cadoles.com/arcad/edge/pkg/module/fetch"
netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
shareSqlite "forge.cadoles.com/arcad/edge/pkg/module/share/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/Masterminds/sprig/v3"
"github.com/go-chi/chi/v5"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
// Register storage drivers
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
)
type Dependencies struct {
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
KeySet jwk.Set
AppRepository appModule.Repository
AppID app.ID
ShareRepository shareModule.Repository
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
ShareStore share.Store
KeySet jwk.Set
AppRepository appModule.Repository
AppID app.ID
}
const defaultSQLiteParams = "?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs *spec.Spec) ([]edgeHTTP.HandlerOptionFunc, error) {
dataDir, err := c.ensureAppDataDir(ctx, appKey)
if err != nil {
return nil, errors.Wrap(err, "could not retrieve app data dir")
appEntry, exists := specs.Apps[appKey]
if !exists {
return nil, errors.Errorf("could not find app entry '%s'", appKey)
}
dbFile := filepath.Join(dataDir, appKey+".sqlite")
db, err := sqlite.Open(dbFile + defaultSQLiteParams)
if err != nil {
return nil, errors.Wrapf(err, "could not open database file '%s'", dbFile)
storage := appEntry.Storage
if storage == nil {
return nil, errors.Errorf("could not find app entry '%s' storage configuration", appKey)
}
shareDBFile := filepath.Join(dataDir, "shared.sqlite")
shareDB, err := sqlite.Open(shareDBFile + defaultSQLiteParams)
documentStore, err := driver.NewDocumentStore(appEntry.Storage.DocumentStoreDSN)
if err != nil {
return nil, errors.Wrapf(err, "could not open database file '%s'", shareDBFile)
return nil, errors.WithStack(err)
}
blobStore, err := driver.NewBlobStore(appEntry.Storage.BlobStoreDSN)
if err != nil {
return nil, errors.WithStack(err)
}
shareStore, err := driver.NewShareStore(appEntry.Storage.ShareStoreDSN)
if err != nil {
return nil, errors.WithStack(err)
}
keySet, err := getAuthKeySet(specs.Config)
@ -79,13 +88,13 @@ func (c *Controller) getHandlerOptions(ctx context.Context, appKey string, specs
mounts = append(mounts, appModule.Mount(c.appRepository))
deps := Dependencies{
Bus: memory.NewBus(),
DocumentStore: sqlite.NewDocumentStoreWithDB(db),
BlobStore: sqlite.NewBlobStoreWithDB(db),
KeySet: keySet,
AppRepository: c.appRepository,
AppID: app.ID(appKey),
ShareRepository: shareSqlite.NewRepositoryWithDB(shareDB),
Bus: memory.NewBus(),
DocumentStore: documentStore,
BlobStore: blobStore,
ShareStore: shareStore,
KeySet: keySet,
AppRepository: c.appRepository,
AppID: app.ID(appKey),
}
modules := c.getAppModules(deps)
@ -293,6 +302,6 @@ func (c *Controller) getAppModules(deps Dependencies) []app.ServerModuleFactory
authModuleFactory(deps.KeySet),
appModule.ModuleFactory(deps.AppRepository),
fetchModule.ModuleFactory(deps.Bus),
shareModule.ModuleFactory(deps.AppID, deps.ShareRepository),
shareModule.ModuleFactory(deps.AppID, deps.ShareStore),
}
}

8
internal/agent/controller/app/auth_module.go
View File

@ -4,7 +4,6 @@ import (
"net/http"
"time"
"forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
appSpec "forge.cadoles.com/Cadoles/emissary/internal/agent/controller/app/spec"
"forge.cadoles.com/Cadoles/emissary/internal/jwk"
"forge.cadoles.com/arcad/edge/pkg/app"
@ -58,7 +57,7 @@ func authModuleFactory(keySet jwk.Set) app.ServerModuleFactory {
)
}
func getAuthMount(auth *spec.Auth, keySet jwk.Set) (auth.MountFunc, error) {
func getAuthMount(auth *appSpec.Auth, keySet jwk.Set) (auth.MountFunc, error) {
switch {
case auth.Local != nil:
var rawKey any = auth.Local.Key
@ -81,7 +80,8 @@ func getAuthMount(auth *spec.Auth, keySet jwk.Set) (auth.MountFunc, error) {
return authModule.Mount(
authHTTP.NewLocalHandler(
jwa.HS256, key,
key,
jwa.HS256,
authHTTP.WithRoutePrefix("/auth"),
authHTTP.WithAccounts(auth.Local.Accounts...),
authHTTP.WithCookieOptions(getCookieDomain, cookieDuration),
@ -111,8 +111,8 @@ func getAnonymousUserMiddleware(auth *appSpec.Auth) (func(http.Handler) http.Han
}
middleware := authModuleMiddleware.AnonymousUser(
anonymousUserSigningKey.Algorithm(),
anonymousUserSigningKey,
auth.Local.SigningAlgorithm,
authModuleMiddleware.WithCookieOptions(getCookieDomain, cookieDuration),
)

29
internal/agent/controller/app/spec/schema.json
View File

@ -26,13 +26,34 @@
"zip",
"tar.gz"
]
},
"storage": {
"type": "object",
"properties": {
"blobStoreDsn": {
"type": "string"
},
"documentStoreDsn": {
"type": "string"
},
"shareStoreDsn": {
"type": "string"
}
},
"required": [
"blobStoreDsn",
"documentStoreDsn",
"shareStoreDsn"
],
"additionalProperties": false
}
},
"required": [
"url",
"sha256sum",
"address",
"format"
"format",
"storage"
],
"additionalProperties": false
}
@ -84,6 +105,9 @@
"key": {
"type": ["object", "string"]
},
"signingAlgorithm": {
"type": "string"
},
"accounts": {
"type": "array",
"items": {
@ -117,7 +141,8 @@
}
},
"required": [
"key"
"key",
"signingAlgorithm"
],
"additionalProperties": false
}

25
internal/agent/controller/app/spec/spec.go
View File

@ -3,6 +3,7 @@ package spec
import (
"forge.cadoles.com/Cadoles/emissary/internal/spec"
edgeAuth "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
"github.com/lestrrat-go/jwx/v2/jwa"
)
const Name spec.Name = "app.emissary.cadoles.com"
@ -14,10 +15,17 @@ type Spec struct {
}
type AppEntry struct {
URL string `json:"url"`
SHA256Sum string `json:"sha256sum"`
Address string `json:"address"`
Format string `json:"format"`
URL string `json:"url"`
SHA256Sum string `json:"sha256sum"`
Address string `json:"address"`
Format string `json:"format"`
Storage *AppStorage `json:"storage"`
}
type AppStorage struct {
ShareStoreDSN string `json:"shareStoreDsn"`
DocumentStoreDSN string `json:"documentStoreDsn"`
BlobStoreDSN string `json:"blobStoreDsn"`
}
type Auth struct {
@ -25,10 +33,11 @@ type Auth struct {
}
type LocalAuth struct {
Key any `json:"key"`
Accounts []edgeAuth.LocalAccount `json:"accounts"`
CookieDomain string `json:"cookieDomain"`
CookieDuration string `json:"cookieDuration"`
Key any `json:"key"`
SigningAlgorithm jwa.SignatureAlgorithm `json:"signingAlgorithm"`
Accounts []edgeAuth.LocalAccount `json:"accounts"`
CookieDomain string `json:"cookieDomain"`
CookieDuration string `json:"cookieDuration"`
}
type Config struct {

8
internal/agent/controller/app/spec/testdata/spec-ok.json vendored
View File

@ -6,7 +6,12 @@
"url": "http://example.com/edge.sdk.client.test_0.0.0.zip",
"sha256sum": "58019192dacdae17755707719707db007e26dac856102280583fbd18427dd352",
"address": ":8081",
"format": "zip"
"format": "zip",
"storage": {
"blobStoreDsn": "sqlite://apps/data/edge.sdk.client.test/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
"shareStoreDsn": "sqlite://apps/data/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
"documentStoreDsn": "sqlite://apps/data/edge.sdk.client.test/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
}
}
},
"config": {
@ -23,6 +28,7 @@
"q": "yJJLNc9w6O4y2icME8k99FugV9E7ObwUxF3v5JN3y1cmAT0h2njyE3iAGqaDZwcY1_jGCisjwoqX6i5E8xqhxX3Gcy3J7SmUAf8fhY8wU3zv9DK7skg2IdvanDb8Y1OM6GchbYZAOVPEg2IvVio8zI-Ih3DDwDk8Df0ufzoHRb8",
"qi": "zOE-4R3cjPesm3MX-4PdwmsaF9QZLUVRUvvHJ08pKs6kAXP18hzjctAoOjhQDxlTYqNYNePfKzKwost3OJoPgRIc9w9qwUCK1gNOS4Z_xozCIaXgMddNFhkoAfZ4JaKjNCiinzjGfqG99Lf-yzmmREuuhRv7SdS3ST4VQjiJQew"
},
"signingAlgorithm": "RS256",
"accounts": [
{
"username": "foo",