arcad/edge
15
0
Fork 0
Code Issues 8 Pull Requests Projects Releases 3 Wiki Activity

feat(storage): rpc based implementation
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
Details

Browse Source
This commit is contained in:
wpetit
2023-09-12 22:03:25 -06:00
parent c3535a4a9b
commit 8e574c299b
113 changed files with 3007 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
cmd/cli/command/app/run.go
View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
@ -28,9 +27,7 @@ import (
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
shareSqlite "forge.cadoles.com/arcad/edge/pkg/module/share/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage"
storageSqlite "forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/bundle"
@ -45,6 +42,12 @@ import (
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/argon2id"
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/plain"
// Register storage drivers
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func RunCommand() *cli.Command {
@ -75,14 +78,22 @@ func RunCommand() *cli.Command {
Value: 0,
},
&cli.StringFlag{
Name: "storage-file",
Usage: "use `FILE` for SQLite storage database",
Value: ".edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
Name: "blobstore-dsn",
Usage: "use `DSN` for blob storage",
EnvVars: []string{"EDGE_BLOBSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "shared-resources-file",
Usage: "use `FILE` for SQLite shared resources database",
Value: ".edge/shared-resources.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
Name: "documentstore-dsn",
Usage: "use `DSN` for document storage",
EnvVars: []string{"EDGE_DOCUMENTSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "sharestore-dsn",
Usage: "use `DSN` for share storage",
EnvVars: []string{"EDGE_SHARESTORE_DSN"},
Value: "sqlite://.edge/share.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "accounts-file",
@ -96,9 +107,10 @@ func RunCommand() *cli.Command {
logFormat := ctx.String("log-format")
logLevel := ctx.Int("log-level")
storageFile := ctx.String("storage-file")
blobstoreDSN := ctx.String("blobstore-dsn")
documentstoreDSN := ctx.String("documentstore-dsn")
shareStoreDSN := ctx.String("sharestore-dsn")
accountsFile := ctx.String("accounts-file")
sharedResourcesFile := ctx.String("shared-resources-file")
logger.SetFormat(logger.Format(logFormat))
logger.SetLevel(logger.Level(logLevel))
@ -144,7 +156,7 @@ func RunCommand() *cli.Command {
appCtx := logger.With(cmdCtx, logger.F("address", address))
if err := runApp(appCtx, path, address, storageFile, accountsFile, appsRepository, sharedResourcesFile); err != nil {
if err := runApp(appCtx, path, address, documentstoreDSN, blobstoreDSN, shareStoreDSN, accountsFile, appsRepository); err != nil {
logger.Error(appCtx, "could not run app", logger.E(errors.WithStack(err)))
}
}(p, port, idx)
@ -157,7 +169,7 @@ func RunCommand() *cli.Command {
}
}
func runApp(ctx context.Context, path string, address string, storageFile string, accountsFile string, appRepository appModule.Repository, sharedResourcesFile string) error {
func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN, shareStoreDSN, accountsFile string, appRepository appModule.Repository) error {
absPath, err := filepath.Abs(path)
if err != nil {
return errors.Wrapf(err, "could not resolve path '%s'", path)
@ -190,9 +202,8 @@ func runApp(ctx context.Context, path string, address string, storageFile string
deps := &moduleDeps{}
funcs := []ModuleDepFunc{
initMemoryBus,
initDatastores(storageFile, manifest.ID),
initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN, manifest.ID),
initAccounts(accountsFile, manifest.ID),
initShareRepository(sharedResourcesFile),
initAppRepository(appRepository),
}
@ -243,13 +254,13 @@ func runApp(ctx context.Context, path string, address string, storageFile string
}
type moduleDeps struct {
AppID app.ID
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
AppRepository appModule.Repository
ShareRepository shareModule.Repository
Accounts []authHTTP.LocalAccount
AppID app.ID
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
AppRepository appModule.Repository
ShareStore share.Store
Accounts []authHTTP.LocalAccount
}
type ModuleDepFunc func(*moduleDeps) error
@ -269,7 +280,7 @@ func getServerModules(deps *moduleDeps) []app.ServerModuleFactory {
),
appModule.ModuleFactory(deps.AppRepository),
fetch.ModuleFactory(deps.Bus),
shareModule.ModuleFactory(deps.AppID, deps.ShareRepository),
shareModule.ModuleFactory(deps.AppID, deps.ShareStore),
}
}
@ -323,10 +334,10 @@ func loadLocalAccounts(path string) ([]authHTTP.LocalAccount, error) {
return nil, errors.WithStack(err)
}
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
if err := ioutil.WriteFile(path, defaultAccounts, 0o640); err != nil {
if err := os.WriteFile(path, defaultAccounts, 0o640); err != nil {
return nil, errors.WithStack(err)
}
@ -437,21 +448,32 @@ func initMemoryBus(deps *moduleDeps) error {
return nil
}
func initDatastores(storageFile string, appID app.ID) ModuleDepFunc {
func initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN string, appID app.ID) ModuleDepFunc {
return func(deps *moduleDeps) error {
storageFile = injectAppID(storageFile, appID)
documentStoreDSN = injectAppID(documentStoreDSN, appID)
if err := ensureDir(storageFile); err != nil {
return errors.WithStack(err)
}
db, err := storageSqlite.Open(storageFile)
documentStore, err := driver.NewDocumentStore(documentStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.DocumentStore = storageSqlite.NewDocumentStoreWithDB(db)
deps.BlobStore = storageSqlite.NewBlobStoreWithDB(db)
deps.DocumentStore = documentStore
blobStoreDSN = injectAppID(blobStoreDSN, appID)
blobStore, err := driver.NewBlobStore(blobStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.BlobStore = blobStore
shareStore, err := driver.NewShareStore(shareStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.ShareStore = shareStore
return nil
}
@ -471,17 +493,3 @@ func initAccounts(accountsFile string, appID app.ID) ModuleDepFunc {
return nil
}
}
func initShareRepository(shareRepositoryFile string) ModuleDepFunc {
return func(deps *moduleDeps) error {
if err := ensureDir(shareRepositoryFile); err != nil {
return errors.WithStack(err)
}
repo := shareSqlite.NewRepository(shareRepositoryFile)
deps.ShareRepository = repo
return nil
}
}