Compare commits

...

10 Commits

Author SHA1 Message Date
c63af872ea feat: goreleaser packaging
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-09-28 14:26:46 -06:00
8e574c299b feat(storage): rpc based implementation
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-09-28 12:36:30 -06:00
c3535a4a9b feat(http): allow passing middlewares via options
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-09-20 09:23:53 -06:00
7e58551f6a docs(context): remove reference to obsolete attribute
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-09-20 09:02:27 -06:00
41d5db6321 docs(auth): add informations about anonymous users
ref arcad/edge-menu#86
2023-09-20 09:01:36 -06:00
8eb441daee feat(auth): automatically generate anonymous user session
All checks were successful
arcad/edge/pipeline/head This commit looks good
ref arcad/edge-menu#86
2023-09-20 08:55:49 -06:00
17808d14c9 fix: prevent bus congestion by flushing out messages
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-26 15:53:23 +02:00
ba9ae6e391 fix(app): use event loop runtime for every operations
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-24 12:16:30 +02:00
abc60b9ae3 fix(module,app): use whole remote address if splitting fail
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-21 20:01:43 +02:00
f99b1ac6ac feat(module,share): cross-app resource sharing module
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-21 12:40:09 +02:00
138 changed files with 5537 additions and 302 deletions

View File

@ -1 +1,3 @@
RUN_APP_ARGS="" RUN_APP_ARGS=""
#EDGE_DOCUMENTSTORE_DSN="rpc://localhost:3001/documentstore?tenant=local&appId=%APPID%"
#EDGE_BLOBSTORE_DSN="rpc://localhost:3001/blobstore?tenant=local&appId=%APPID%"

5
.gitignore vendored
View File

@ -5,3 +5,8 @@
*.sqlite *.sqlite
/.gitea-release /.gitea-release
/.edge /.edge
/data
.mktools/
/dist
/.chglog
/CHANGELOG.md

103
.goreleaser.yml Normal file
View File

@ -0,0 +1,103 @@
project_name: edge
before:
hooks:
- go mod tidy
- go generate ./...
builds:
- id: edge-cli
binary: edge-cli
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
main: ./cmd/cli
- id: storage-server
binary: storage-server
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
main: ./cmd/storage-server
archives:
- id: edge-cli
builds: ["edge-cli"]
name_template: 'edge-cli_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
- id: storage-server
builds: ["storage-server"]
name_template: 'storage-server_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ .Version }}"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
nfpms:
- id: edge-cli
builds:
- "edge-cli"
package_name: edge-cli
homepage: https://forge.cadoles.com/arcad/edge
maintainer: William Petit <wpetit@cadoles.com>
description: |-
license: AGPL-3.0
formats:
- apk
- deb
- id: storage-server
builds:
- "storage-server"
package_name: storage-server
homepage: https://forge.cadoles.com/arcad/edge
maintainer: William Petit <wpetit@cadoles.com>
description: |-
license: AGPL-3.0
formats:
- apk
- deb
# contents:
# - src: misc/packaging/common/config-agent.yml
# dst: /etc/emissary/agent.yml
# type: config
# - src: misc/packaging/systemd/emissary-agent.systemd.service
# dst: /usr/lib/systemd/system/emissary-agent.service
# packager: deb
# - src: misc/packaging/systemd/emissary-agent.systemd.service
# dst: /usr/lib/systemd/system/emissary-agent.service
# packager: rpm
# - src: misc/packaging/openrc/emissary-agent.openrc.sh
# dst: /etc/init.d/emissary-agent
# file_info:
# mode: 0755
# packager: apk
# scripts:
# postinstall: "misc/packaging/common/postinstall-agent.sh"

View File

@ -6,14 +6,17 @@ GOTEST_ARGS ?= -short -timeout 60s
ESBUILD_VERSION ?= v0.17.5 ESBUILD_VERSION ?= v0.17.5
GIT_VERSION := $(shell git describe --always)
DATE_VERSION := $(shell date +%Y.%-m.%-d)
FULL_VERSION := v$(DATE_VERSION)-$(GIT_VERSION)$(if $(shell git diff --stat),-dirty,)
APP_PATH ?= misc/client-sdk-testsuite/dist APP_PATH ?= misc/client-sdk-testsuite/dist
RUN_APP_ARGS ?= RUN_APP_ARGS ?=
RUN_STORAGE_SERVER_ARGS ?=
GORELEASER_VERSION ?= v1.21.2
GORELEASER_ARGS ?= release --snapshot --clean
SHELL := bash SHELL := bash
build: build-edge-cli build-client-sdk-test-app
build: build-cli build-storage-server build-client-sdk-test-app
watch: tools/modd/bin/modd watch: tools/modd/bin/modd
tools/modd/bin/modd tools/modd/bin/modd
@ -22,17 +25,23 @@ watch: tools/modd/bin/modd
test: test-go test: test-go
test-go: test-go:
go test -v -count=1 $(GOTEST_ARGS) ./... go test -count=1 $(GOTEST_ARGS) ./...
lint: lint:
golangci-lint run --enable-all $(LINT_ARGS) golangci-lint run --enable-all $(LINT_ARGS)
build-edge-cli: build-sdk build-cli: build-sdk
CGO_ENABLED=0 go build \ CGO_ENABLED=0 go build \
-v \ -v \
-o ./bin/cli \ -o ./bin/cli \
./cmd/cli ./cmd/cli
build-storage-server: build-sdk
CGO_ENABLED=0 go build \
-v \
-o ./bin/storage-server \
./cmd/storage-server
build-client-sdk-test-app: build-client-sdk-test-app:
cd misc/client-sdk-testsuite && $(MAKE) dist cd misc/client-sdk-testsuite && $(MAKE) dist
@ -68,25 +77,31 @@ node_modules:
run-app: .env run-app: .env
( set -o allexport && source .env && set +o allexport && bin/cli app run -p $(APP_PATH) $$RUN_APP_ARGS ) ( set -o allexport && source .env && set +o allexport && bin/cli app run -p $(APP_PATH) $$RUN_APP_ARGS )
run-storage-server: .env
( set -o allexport && source .env && set +o allexport && bin/storage-server run $$RUN_STORAGE_SERVER_ARGS )
.env: .env:
cp .env.dist .env cp .env.dist .env
gitea-release: tools/yq/bin/yq tools/gitea-release/bin/gitea-release.sh build gitea-release: tools/yq/bin/yq tools/gitea-release/bin/gitea-release.sh goreleaser build
mkdir -p .gitea-release mkdir -p .gitea-release
rm -rf .gitea-release/* rm -rf .gitea-release/*
cp bin/cli .gitea-release/edge_cli_amd64 cp dist/*.deb .gitea-release/
cp dist/*.tar.gz .gitea-release/
cp dist/*.apk .gitea-release/
cp CHANGELOG.md .gitea-release/
# Create client-sdk-testsuite package # Create client-sdk-testsuite package
tools/yq/bin/yq -i '.version = "$(FULL_VERSION)"' ./misc/client-sdk-testsuite/dist/manifest.yml tools/yq/bin/yq -i '.version = "$(MKT_PROJECT_VERSION)"' ./misc/client-sdk-testsuite/dist/manifest.yml
.gitea-release/edge_cli_amd64 app package -d ./misc/client-sdk-testsuite/dist -o .gitea-release bin/cli app package -d ./misc/client-sdk-testsuite/dist -o .gitea-release
GITEA_RELEASE_PROJECT="edge" \ GITEA_RELEASE_PROJECT="edge" \
GITEA_RELEASE_ORG="arcad" \ GITEA_RELEASE_ORG="arcad" \
GITEA_RELEASE_BASE_URL="https://forge.cadoles.com" \ GITEA_RELEASE_BASE_URL="https://forge.cadoles.com" \
GITEA_RELEASE_VERSION="$(FULL_VERSION)" \ GITEA_RELEASE_VERSION="$(MKT_PROJECT_VERSION)" \
GITEA_RELEASE_NAME="$(FULL_VERSION)" \ GITEA_RELEASE_NAME="$(MKT_PROJECT_VERSION)" \
GITEA_RELEASE_COMMITISH_TARGET="$(GIT_VERSION)" \ GITEA_RELEASE_COMMITISH_TARGET="$$(git rev-parse HEAD)" \
GITEA_RELEASE_IS_DRAFT="false" \ GITEA_RELEASE_IS_DRAFT="false" \
GITEA_RELEASE_IS_PRERELEASE="true" \ GITEA_RELEASE_IS_PRERELEASE="true" \
GITEA_RELEASE_BODY="" \ GITEA_RELEASE_BODY="" \
@ -106,3 +121,21 @@ tools/yq/bin/yq:
tools/modd/bin/modd: tools/modd/bin/modd:
mkdir -p tools/modd/bin mkdir -p tools/modd/bin
GOBIN=$(PWD)/tools/modd/bin go install -mod=readonly github.com/cortesi/modd/cmd/modd@latest GOBIN=$(PWD)/tools/modd/bin go install -mod=readonly github.com/cortesi/modd/cmd/modd@latest
.PHONY: goreleaser
goreleaser: .mktools changelog
( set -o allexport && source .env && set +o allexport && curl -sfL https://goreleaser.com/static/run | VERSION="$(GORELEASER_VERSION)" GORELEASER_CURRENT_TAG="$(MKT_PROJECT_VERSION)" bash /dev/stdin $(GORELEASER_ARGS) )
.PHONY: changelog
changelog: .mktools
$(MAKE) MKT_GIT_CHGLOG_ARGS='--next-tag $$(MKT_PROJECT_VERSION) --tag-filter-pattern $$(MKT_PROJECT_VERSION_CHANNEL) --output CHANGELOG.md' mkt-changelog
.PHONY: mktools
mktools:
rm -rf .mktools
curl -q https://forge.cadoles.com/Cadoles/mktools/raw/branch/master/install.sh | $(SHELL)
.mktools:
$(MAKE) mktools
-include .mktools/*.mk

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"os" "os"
@ -22,12 +21,13 @@ import (
appModuleMemory "forge.cadoles.com/arcad/edge/pkg/module/app/memory" appModuleMemory "forge.cadoles.com/arcad/edge/pkg/module/app/memory"
authModule "forge.cadoles.com/arcad/edge/pkg/module/auth" authModule "forge.cadoles.com/arcad/edge/pkg/module/auth"
authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http" authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
authModuleMiddleware "forge.cadoles.com/arcad/edge/pkg/module/auth/middleware"
"forge.cadoles.com/arcad/edge/pkg/module/blob" "forge.cadoles.com/arcad/edge/pkg/module/blob"
"forge.cadoles.com/arcad/edge/pkg/module/cast" "forge.cadoles.com/arcad/edge/pkg/module/cast"
"forge.cadoles.com/arcad/edge/pkg/module/fetch" "forge.cadoles.com/arcad/edge/pkg/module/fetch"
netModule "forge.cadoles.com/arcad/edge/pkg/module/net" netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
"forge.cadoles.com/arcad/edge/pkg/storage" "forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"gitlab.com/wpetit/goweb/logger" "gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/bundle" "forge.cadoles.com/arcad/edge/pkg/bundle"
@ -42,6 +42,12 @@ import (
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/argon2id" _ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/argon2id"
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/plain" _ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/plain"
// Register storage drivers
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
) )
func RunCommand() *cli.Command { func RunCommand() *cli.Command {
@ -72,9 +78,22 @@ func RunCommand() *cli.Command {
Value: 0, Value: 0,
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "storage-file", Name: "blobstore-dsn",
Usage: "use `FILE` for SQLite storage database", Usage: "use `DSN` for blob storage",
Value: ".edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000", EnvVars: []string{"EDGE_BLOBSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "documentstore-dsn",
Usage: "use `DSN` for document storage",
EnvVars: []string{"EDGE_DOCUMENTSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "sharestore-dsn",
Usage: "use `DSN` for share storage",
EnvVars: []string{"EDGE_SHARESTORE_DSN"},
Value: "sqlite://.edge/share.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
}, },
&cli.StringFlag{ &cli.StringFlag{
Name: "accounts-file", Name: "accounts-file",
@ -88,7 +107,9 @@ func RunCommand() *cli.Command {
logFormat := ctx.String("log-format") logFormat := ctx.String("log-format")
logLevel := ctx.Int("log-level") logLevel := ctx.Int("log-level")
storageFile := ctx.String("storage-file") blobstoreDSN := ctx.String("blobstore-dsn")
documentstoreDSN := ctx.String("documentstore-dsn")
shareStoreDSN := ctx.String("sharestore-dsn")
accountsFile := ctx.String("accounts-file") accountsFile := ctx.String("accounts-file")
logger.SetFormat(logger.Format(logFormat)) logger.SetFormat(logger.Format(logFormat))
@ -135,7 +156,7 @@ func RunCommand() *cli.Command {
appCtx := logger.With(cmdCtx, logger.F("address", address)) appCtx := logger.With(cmdCtx, logger.F("address", address))
if err := runApp(appCtx, path, address, storageFile, accountsFile, appsRepository); err != nil { if err := runApp(appCtx, path, address, documentstoreDSN, blobstoreDSN, shareStoreDSN, accountsFile, appsRepository); err != nil {
logger.Error(appCtx, "could not run app", logger.E(errors.WithStack(err))) logger.Error(appCtx, "could not run app", logger.E(errors.WithStack(err)))
} }
}(p, port, idx) }(p, port, idx)
@ -148,7 +169,7 @@ func RunCommand() *cli.Command {
} }
} }
func runApp(ctx context.Context, path string, address string, storageFile string, accountsFile string, appRepository appModule.Repository) error { func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN, shareStoreDSN, accountsFile string, appRepository appModule.Repository) error {
absPath, err := filepath.Abs(path) absPath, err := filepath.Abs(path)
if err != nil { if err != nil {
return errors.Wrapf(err, "could not resolve path '%s'", path) return errors.Wrapf(err, "could not resolve path '%s'", path)
@ -172,48 +193,45 @@ func runApp(ctx context.Context, path string, address string, storageFile string
ctx = logger.With(ctx, logger.F("appID", manifest.ID)) ctx = logger.With(ctx, logger.F("appID", manifest.ID))
storageFile = injectAppID(storageFile, manifest.ID)
if err := ensureDir(storageFile); err != nil {
return errors.WithStack(err)
}
db, err := sqlite.Open(storageFile)
if err != nil {
return errors.WithStack(err)
}
accountsFile = injectAppID(accountsFile, manifest.ID)
accounts, err := loadLocalAccounts(accountsFile)
if err != nil {
return errors.Wrap(err, "could not load local accounts")
}
// Add auth handler // Add auth handler
key, err := dummyKey() key, err := dummyKey()
if err != nil { if err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
ds := sqlite.NewDocumentStoreWithDB(db) deps := &moduleDeps{}
bs := sqlite.NewBlobStoreWithDB(db) funcs := []ModuleDepFunc{
bus := memory.NewBus() initMemoryBus,
initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN, manifest.ID),
initAccounts(accountsFile, manifest.ID),
initAppRepository(appRepository),
}
for _, fn := range funcs {
if err := fn(deps); err != nil {
return errors.WithStack(err)
}
}
handler := appHTTP.NewHandler( handler := appHTTP.NewHandler(
appHTTP.WithBus(bus), appHTTP.WithBus(deps.Bus),
appHTTP.WithServerModules(getServerModules(bus, ds, bs, appRepository)...), appHTTP.WithServerModules(getServerModules(deps)...),
appHTTP.WithHTTPMounts( appHTTP.WithHTTPMounts(
appModule.Mount(appRepository), appModule.Mount(appRepository),
authModule.Mount( authModule.Mount(
authHTTP.NewLocalHandler( authHTTP.NewLocalHandler(
jwa.HS256, key, jwa.HS256, key,
authHTTP.WithRoutePrefix("/auth"), authHTTP.WithRoutePrefix("/auth"),
authHTTP.WithAccounts(accounts...), authHTTP.WithAccounts(deps.Accounts...),
), ),
authModule.WithJWT(dummyKeySet), authModule.WithJWT(dummyKeySet),
), ),
), ),
appHTTP.WithHTTPMiddlewares(
authModuleMiddleware.AnonymousUser(
jwa.HS256, key,
),
),
) )
if err := handler.Load(bundle); err != nil { if err := handler.Load(bundle); err != nil {
return errors.Wrap(err, "could not load app bundle") return errors.Wrap(err, "could not load app bundle")
@ -235,21 +253,34 @@ func runApp(ctx context.Context, path string, address string, storageFile string
return nil return nil
} }
func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStore, appRepository appModule.Repository) []app.ServerModuleFactory { type moduleDeps struct {
AppID app.ID
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
AppRepository appModule.Repository
ShareStore share.Store
Accounts []authHTTP.LocalAccount
}
type ModuleDepFunc func(*moduleDeps) error
func getServerModules(deps *moduleDeps) []app.ServerModuleFactory {
return []app.ServerModuleFactory{ return []app.ServerModuleFactory{
module.LifecycleModuleFactory(),
module.ContextModuleFactory(), module.ContextModuleFactory(),
module.ConsoleModuleFactory(), module.ConsoleModuleFactory(),
cast.CastModuleFactory(), cast.CastModuleFactory(),
module.LifecycleModuleFactory(), netModule.ModuleFactory(deps.Bus),
netModule.ModuleFactory(bus), module.RPCModuleFactory(deps.Bus),
module.RPCModuleFactory(bus), module.StoreModuleFactory(deps.DocumentStore),
module.StoreModuleFactory(ds), blob.ModuleFactory(deps.Bus, deps.BlobStore),
blob.ModuleFactory(bus, bs),
authModule.ModuleFactory( authModule.ModuleFactory(
authModule.WithJWT(dummyKeySet), authModule.WithJWT(dummyKeySet),
), ),
appModule.ModuleFactory(appRepository), appModule.ModuleFactory(deps.AppRepository),
fetch.ModuleFactory(bus), fetch.ModuleFactory(deps.Bus),
shareModule.ModuleFactory(deps.AppID, deps.ShareStore),
} }
} }
@ -303,10 +334,10 @@ func loadLocalAccounts(path string) ([]authHTTP.LocalAccount, error) {
return nil, errors.WithStack(err) return nil, errors.WithStack(err)
} }
data, err := ioutil.ReadFile(path) data, err := os.ReadFile(path)
if err != nil { if err != nil {
if errors.Is(err, os.ErrNotExist) { if errors.Is(err, os.ErrNotExist) {
if err := ioutil.WriteFile(path, defaultAccounts, 0o640); err != nil { if err := os.WriteFile(path, defaultAccounts, 0o640); err != nil {
return nil, errors.WithStack(err) return nil, errors.WithStack(err)
} }
@ -403,3 +434,62 @@ func newAppRepository(host string, basePort uint64, manifests ...*app.Manifest)
manifests..., manifests...,
) )
} }
func initAppRepository(repo appModule.Repository) ModuleDepFunc {
return func(deps *moduleDeps) error {
deps.AppRepository = repo
return nil
}
}
func initMemoryBus(deps *moduleDeps) error {
deps.Bus = memory.NewBus()
return nil
}
func initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN string, appID app.ID) ModuleDepFunc {
return func(deps *moduleDeps) error {
documentStoreDSN = injectAppID(documentStoreDSN, appID)
documentStore, err := driver.NewDocumentStore(documentStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.DocumentStore = documentStore
blobStoreDSN = injectAppID(blobStoreDSN, appID)
blobStore, err := driver.NewBlobStore(blobStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.BlobStore = blobStore
shareStore, err := driver.NewShareStore(shareStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.ShareStore = shareStore
return nil
}
}
func initAccounts(accountsFile string, appID app.ID) ModuleDepFunc {
return func(deps *moduleDeps) error {
accountsFile = injectAppID(accountsFile, appID)
accounts, err := loadLocalAccounts(accountsFile)
if err != nil {
return errors.Wrap(err, "could not load local accounts")
}
deps.Accounts = accounts
return nil
}
}

View File

@ -0,0 +1,13 @@
package auth
import (
"github.com/urfave/cli/v2"
)
func Root() *cli.Command {
return &cli.Command{
Name: "auth",
Usage: "Auth related command",
Subcommands: []*cli.Command{},
}
}

View File

@ -0,0 +1,48 @@
package command
import (
"context"
"fmt"
"os"
"sort"
"github.com/urfave/cli/v2"
)
func Main(commands ...*cli.Command) {
ctx := context.Background()
app := &cli.App{
Name: "storage-server",
Usage: "Edge storage server",
Commands: commands,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "debug",
EnvVars: []string{"DEBUG"},
Value: false,
},
},
}
app.ExitErrHandler = func(ctx *cli.Context, err error) {
if err == nil {
return
}
debug := ctx.Bool("debug")
if !debug {
fmt.Printf("[ERROR] %v\n", err)
} else {
fmt.Printf("%+v", err)
}
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
if err := app.RunContext(ctx, os.Args); err != nil {
os.Exit(1)
}
}

View File

@ -0,0 +1,179 @@
package command
import (
"fmt"
"net/http"
"strings"
"sync"
"time"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/keegancsmith/rpc"
"gitlab.com/wpetit/goweb/logger"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
// Register storage drivers
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func Run() *cli.Command {
return &cli.Command{
Name: "run",
Usage: "Run server",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "address",
Aliases: []string{"addr"},
Value: ":3001",
},
&cli.StringFlag{
Name: "blobstore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_BLOBSTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
},
&cli.StringFlag{
Name: "documentstore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_DOCUMENTSTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
},
&cli.StringFlag{
Name: "sharestore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_SHARESTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
},
&cli.DurationFlag{
Name: "cache-ttl",
EnvVars: []string{"STORAGE_SERVER_CACHE_TTL"},
Value: time.Hour,
},
&cli.IntFlag{
Name: "cache-size",
EnvVars: []string{"STORAGE_SERVER_CACHE_SIZE"},
Value: 32,
},
},
Action: func(ctx *cli.Context) error {
addr := ctx.String("address")
blobStoreDSNPattern := ctx.String("blobstore-dsn-pattern")
documentStoreDSNPattern := ctx.String("documentstore-dsn-pattern")
shareStoreDSNPattern := ctx.String("sharestore-dsn-pattern")
cacheSize := ctx.Int("cache-size")
cacheTTL := ctx.Duration("cache-ttl")
router := chi.NewRouter()
getBlobStoreServer := createGetCachedStoreServer(
func(dsn string) (storage.BlobStore, error) {
return driver.NewBlobStore(dsn)
},
func(store storage.BlobStore) *rpc.Server {
return server.NewBlobStoreServer(store)
},
)
getShareStoreServer := createGetCachedStoreServer(
func(dsn string) (share.Store, error) {
return driver.NewShareStore(dsn)
},
func(store share.Store) *rpc.Server {
return server.NewShareStoreServer(store)
},
)
getDocumentStoreServer := createGetCachedStoreServer(
func(dsn string) (storage.DocumentStore, error) {
return driver.NewDocumentStore(dsn)
},
func(store storage.DocumentStore) *rpc.Server {
return server.NewDocumentStoreServer(store)
},
)
router.Use(middleware.RealIP)
router.Use(middleware.Logger)
router.Handle("/blobstore", createStoreHandler(getBlobStoreServer, blobStoreDSNPattern, cacheSize, cacheTTL))
router.Handle("/documentstore", createStoreHandler(getDocumentStoreServer, documentStoreDSNPattern, cacheSize, cacheTTL))
router.Handle("/sharestore", createStoreHandler(getShareStoreServer, shareStoreDSNPattern, cacheSize, cacheTTL))
if err := http.ListenAndServe(addr, router); err != nil {
return errors.WithStack(err)
}
return nil
},
}
}
type getRPCServerFunc func(cacheSize int, cacheTTL time.Duration, tenant, appID, dsnPattern string) (*rpc.Server, error)
func createGetCachedStoreServer[T any](storeFactory func(dsn string) (T, error), serverFactory func(store T) *rpc.Server) getRPCServerFunc {
var (
cache *expirable.LRU[string, *rpc.Server]
initCache sync.Once
)
return func(cacheSize int, cacheTTL time.Duration, tenant, appID, dsnPattern string) (*rpc.Server, error) {
initCache.Do(func() {
cache = expirable.NewLRU[string, *rpc.Server](cacheSize, nil, cacheTTL)
})
key := fmt.Sprintf("%s:%s", tenant, appID)
storeServer, _ := cache.Get(key)
if storeServer != nil {
return storeServer, nil
}
dsn := strings.ReplaceAll(dsnPattern, "%TENANT%", tenant)
dsn = strings.ReplaceAll(dsn, "%APPID%", appID)
store, err := storeFactory(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
storeServer = serverFactory(store)
cache.Add(key, storeServer)
return storeServer, nil
}
}
func createStoreHandler(getStoreServer getRPCServerFunc, dsnPattern string, cacheSize int, cacheTTL time.Duration) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
tenant := r.URL.Query().Get("tenant")
if tenant == "" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
appID := r.URL.Query().Get("appId")
if tenant == "" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
server, err := getStoreServer(cacheSize, cacheTTL, tenant, appID, dsnPattern)
if err != nil {
logger.Error(r.Context(), "could not retrieve store server", logger.E(errors.WithStack(err)), logger.F("tenant", tenant))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
server.ServeHTTP(w, r)
})
}

View File

@ -0,0 +1,13 @@
package main
import (
"forge.cadoles.com/arcad/edge/cmd/storage-server/command"
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/auth"
)
func main() {
command.Main(
command.Run(),
auth.Root(),
)
}

View File

@ -29,4 +29,5 @@ Listes des modules disponibles côté serveur.
- [`fetch`](./fetch.md) - [`fetch`](./fetch.md)
- [`net`](./net.md) - [`net`](./net.md)
- [`rpc`](./rpc.md) - [`rpc`](./rpc.md)
- [`share`](./share.md)
- [`store`](./store.md) - [`store`](./store.md)

View File

@ -2,6 +2,14 @@
Ce module permet de récupérer des informations concernant l'utilisateur connecté et ses attributs. Ce module permet de récupérer des informations concernant l'utilisateur connecté et ses attributs.
### Utilisateurs anonymes
Edge génère automatiquement une session pour les utilisateurs anonymes. Ainsi, qu'un utilisateur soit identifié ou non les `claims` suivants seront toujours valués:
- `auth.CLAIM_SUBJECT`
- `auth.CLAIM_PREFERRED_USERNAME`
- `auth.CLAIM_ISSUER` (prendra la valeur `anon` dans le cas d'un utilisateur anonyme)
## Méthodes ## Méthodes
### `auth.getClaim(ctx: Context, name: string): string` ### `auth.getClaim(ctx: Context, name: string): string`

View File

@ -43,12 +43,6 @@ function onClientMessage(ctx, message) {
} }
``` ```
## Propriétés
### `context.SESSION_ID`
Clé permettant de récupérer la clé de session associé au client émetteur du message courant.
#### Usage #### Usage
```js ```js

View File

@ -0,0 +1,143 @@
# Module `share`
Ce module permet partager des ressources à destination des autres applications exécutées dans l'environnement Edge.
## Propriétés
### `share.ANY_TYPE`, `share.ANY_NAME`
Les propriétés `share.ANY_TYPE` et `share.ANY_NAME` sont utilisées dans la méthode `share.findResources()` pour récupérer ne pas appliquer de filtre spécifique au type ou au nom des attributs respectivement.
### `share.TYPE_TEXT`, `share.TYPE_NUMBER`, `share.TYPE_PATH`, `share.TYPE_BOOL`
Ces propriétés correspondant aux types d'attributs possibles dans une ressource.
Le type `share.TYPE_PATH` décrit un "chemin" destiné à être transformé en URL par l'application consommatrice de la ressource sous la forme `${APP_URL}${PATH}`. Ce type d'attribut peut être utilisé pour partager des URLs (médias, pages, etc) entre applications.
## Méthodes
### `share.upsertResource(ctx: Context, resourceId: string, ...attributes: Attribute[]): Resource`
Cette méthode permet de créer une ressource ou de la mettre à jour si elle existe déjà. Elle prend en paramètre le contexte d'exécution, l'identifiant de la ressource et une liste d'attributs.
Si la ressource n'existe pas, elle sera créée avec les attributs fournis. Si elle existe, les attributs existants seront mis à jour avec les valeurs fournies.
#### Arguments
- `ctx: Context`: Le contexte d'exécution.
- `resourceId: string`: L'identifiant de la ressource.
- `...attributes: Attribute[]`: Une liste d'attributs. Chaque attribut est représenté par un objet de type `Attribute`.
#### Valeur de retour
La méthode retourne un objet de type `Resource` qui représente la ressource créée ou mise à jour.
#### Usage
```javascript
const resource = share.upsertResource(ctx, "my-resource", { name: "color", type: share.TYPE_TEXT, value: "red" });
console.log(resource);
// Output: { id: "my-resource", origin: "my.app", attributes: [{ name: "color", type: "text", value: "red", createdAt: "2023-04-21T14:30:00Z", updatedAt: "2023-04-21T14:30:00Z" }] }
```
### `share.findResources(ctx: Context, withAttribute?: string, withType?: string): []Resource`
Cette méthode permet de rechercher des ressources en fonction de leurs attributs. Elle prend en paramètre le contexte d'exécution et deux paramètres optionnels qui permettent de filtrer les ressources.
#### Arguments
- `ctx: Context`: Le contexte d'exécution.
- `withAttribute?: string`: (optionnel) Le nom de l'attribut à rechercher (`share.ANY_NAME` par défaut)
- `withType?: string`: (optionnel) Le type de l'attribut à rechercher (`share.ANY_TYPE` par défaut)
#### Valeur de retour
La méthode retourne un tableau d'objets de type `Resource` qui représentent les ressources correspondant aux critères de recherche.
#### Usage
```typescript
const resources = share.findResources(ctx, "color", share.TYPE_TEXT);
console.log(resources);
// Output: [{ id: "my-resource", origin: "my/app", attributes: [{ name: "color", type: "text", value: "red", createdAt: "2023-04-21T14:30:00Z", updatedAt: "2023-04-21T14:30:00Z" }] }]
```
### `share.deleteAttributes(ctx: Context, resourceId: string, ...names: string[]): Resource`
Cette méthode supprime un ou plusieurs attributs de la ressource spécifiée.
#### Arguments
- `ctx: Context`: contexte d'exécution
- `resourceId: string`: identifiant unique de la ressource à modifier
- `...names: string[]`: tableau de noms d'attributs à supprimer
#### Valeur de retour
La méthode retourne un objet de type `Resource` qui représente la ressource modifiée.
#### Usage
```typescript
const resource = share.upsertResource(ctx, "my-resource", { name: "color", type: share.TYPE_TEXT, value: "red" });
console.log(resource);
// Output: { id: "my-resource", origin: "my.app", attributes: [{ name: "color", type: "text", value: "red", createdAt: "2023-04-21T14:30:00Z", updatedAt: "2023-04-21T14:30:00Z" }] }
```
### `share.deleteResource(ctx: Context, resourceId: string)`
Cette méthode supprime la ressource spécifiée.
#### Arguments
- `ctx: Context`: contexte d'exécution
- `resourceId: string`: identifiant unique de la ressource à supprimer
#### Valeur de retour
La méthode ne retourne pas de valeur.
#### Usage
```typescript
const resource = share.deleteResource(ctx, "my-resource");
```
## Objets
### `Context`
Voir la documentation du module [`context`](./context.md)
### `Resource`
```typescript
interface Resource {
id: string
origin: string
attributes: Attribute[]
}
```
### `Attribute`
```typescript
interface Attribute {
name: string
type: ValueType
createdAt: string
updatedAt: string
}
```
### `ValueType`
```typescript
enum ValueType {
TYPE_TEXT = "text",
TYPE_PATH = "path",
TYPE_NUMBER = "number",
TYPE_BOOL = "bool"
}
```

4
go.mod
View File

@ -3,7 +3,9 @@ module forge.cadoles.com/arcad/edge
go 1.19 go 1.19
require ( require (
github.com/hashicorp/golang-lru/v2 v2.0.6
github.com/hashicorp/mdns v1.0.5 github.com/hashicorp/mdns v1.0.5
github.com/keegancsmith/rpc v1.3.0
github.com/lestrrat-go/jwx/v2 v2.0.8 github.com/lestrrat-go/jwx/v2 v2.0.8
modernc.org/sqlite v1.20.4 modernc.org/sqlite v1.20.4
) )
@ -43,7 +45,7 @@ require (
github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/chi/v5 v5.0.8
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/google/uuid v1.3.0 // indirect github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.4.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect
github.com/igm/sockjs-go/v3 v3.0.2 github.com/igm/sockjs-go/v3 v3.0.2
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect

4
go.sum
View File

@ -188,6 +188,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM=
github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0= github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0=
github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE= github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE=
@ -201,6 +203,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk=
github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=

View File

@ -9,6 +9,7 @@ modd.conf
prep: make build-client-sdk-test-app prep: make build-client-sdk-test-app
prep: make build prep: make build
daemon: make run-app daemon: make run-app
daemon: make run-storage-server
} }
**/*.go { **/*.go {

View File

@ -39,3 +39,14 @@ func NewPromiseProxy(promise *goja.Promise, resolve func(result interface{}), re
return proxy return proxy
} }
func NewPromiseProxyFrom(rt *goja.Runtime) *PromiseProxy {
promise, resolve, reject := rt.NewPromise()
return NewPromiseProxy(promise, resolve, reject)
}
func IsPromise(v goja.Value) (*goja.Promise, bool) {
promise, ok := v.Export().(*goja.Promise)
return promise, ok
}

View File

@ -17,15 +17,22 @@ var (
) )
type Server struct { type Server struct {
runtime *goja.Runtime
loop *eventloop.EventLoop loop *eventloop.EventLoop
factories []ServerModuleFactory
modules []ServerModule modules []ServerModule
} }
func (s *Server) Load(name string, src string) error { func (s *Server) Load(name string, src string) error {
_, err := s.runtime.RunScript(name, src) var err error
s.loop.RunOnLoop(func(rt *goja.Runtime) {
_, err = rt.RunScript(name, src)
if err != nil { if err != nil {
return errors.Wrap(err, "could not run js script") err = errors.Wrap(err, "could not run js script")
}
})
if err != nil {
return errors.WithStack(err)
} }
return nil return nil
@ -34,15 +41,15 @@ func (s *Server) Load(name string, src string) error {
func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...interface{}) (goja.Value, error) { func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...interface{}) (goja.Value, error) {
ctx = logger.With(ctx, logger.F("function", funcName), logger.F("args", args)) ctx = logger.With(ctx, logger.F("function", funcName), logger.F("args", args))
callable, ok := goja.AssertFunction(s.runtime.Get(funcName)) ret, err := s.Exec(ctx, funcName, args...)
if !ok { if err != nil {
return nil, errors.WithStack(ErrFuncDoesNotExist) return nil, errors.WithStack(err)
} }
return s.Exec(ctx, callable, args...) return ret, nil
} }
func (s *Server) Exec(ctx context.Context, callable goja.Callable, args ...interface{}) (goja.Value, error) { func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...interface{}) (goja.Value, error) {
var ( var (
wg sync.WaitGroup wg sync.WaitGroup
value goja.Value value goja.Value
@ -51,7 +58,28 @@ func (s *Server) Exec(ctx context.Context, callable goja.Callable, args ...inter
wg.Add(1) wg.Add(1)
s.loop.RunOnLoop(func(vm *goja.Runtime) { s.loop.RunOnLoop(func(rt *goja.Runtime) {
var callable goja.Callable
switch typ := callableOrFuncname.(type) {
case goja.Callable:
callable = typ
case string:
call, ok := goja.AssertFunction(rt.Get(typ))
if !ok {
err = errors.WithStack(ErrFuncDoesNotExist)
return
}
callable = call
default:
err = errors.Errorf("callableOrFuncname: expected callable or function name, got '%T'", callableOrFuncname)
return
}
logger.Debug(ctx, "executing callable") logger.Debug(ctx, "executing callable")
defer wg.Done() defer wg.Done()
@ -73,7 +101,7 @@ func (s *Server) Exec(ctx context.Context, callable goja.Callable, args ...inter
jsArgs := make([]goja.Value, 0, len(args)) jsArgs := make([]goja.Value, 0, len(args))
for _, a := range args { for _, a := range args {
jsArgs = append(jsArgs, vm.ToValue(a)) jsArgs = append(jsArgs, rt.ToValue(a))
} }
value, err = callable(nil, jsArgs...) value, err = callable(nil, jsArgs...)
@ -84,12 +112,11 @@ func (s *Server) Exec(ctx context.Context, callable goja.Callable, args ...inter
wg.Wait() wg.Wait()
return value, err if err != nil {
} return nil, errors.WithStack(err)
}
func (s *Server) IsPromise(v goja.Value) (*goja.Promise, bool) { return value, nil
promise, ok := v.Export().(*goja.Promise)
return promise, ok
} }
func (s *Server) WaitForPromise(promise *goja.Promise) goja.Value { func (s *Server) WaitForPromise(promise *goja.Promise) goja.Value {
@ -135,28 +162,21 @@ func (s *Server) WaitForPromise(promise *goja.Promise) goja.Value {
return value return value
} }
func (s *Server) NewPromise() *PromiseProxy {
promise, resolve, reject := s.runtime.NewPromise()
return NewPromiseProxy(promise, resolve, reject)
}
func (s *Server) ToValue(v interface{}) goja.Value {
return s.runtime.ToValue(v)
}
func (s *Server) Start() error { func (s *Server) Start() error {
s.loop.Start() s.loop.Start()
for _, mod := range s.modules { var err error
initMod, ok := mod.(InitializableModule)
if !ok {
continue
}
if err := initMod.OnInit(); err != nil { s.loop.RunOnLoop(func(rt *goja.Runtime) {
return errors.WithStack(err) rt.SetFieldNameMapper(goja.TagFieldNameMapper("goja", true))
rt.SetRandSource(createRandomSource())
if err = s.initModules(rt); err != nil {
err = errors.WithStack(err)
} }
})
if err != nil {
return errors.WithStack(err)
} }
return nil return nil
@ -166,36 +186,46 @@ func (s *Server) Stop() {
s.loop.Stop() s.loop.Stop()
} }
func (s *Server) initModules(factories ...ServerModuleFactory) { func (s *Server) initModules(rt *goja.Runtime) error {
runtime := goja.New() modules := make([]ServerModule, 0, len(s.factories))
runtime.SetFieldNameMapper(goja.TagFieldNameMapper("goja", true)) for _, moduleFactory := range s.factories {
runtime.SetRandSource(createRandomSource())
modules := make([]ServerModule, 0, len(factories))
for _, moduleFactory := range factories {
mod := moduleFactory(s) mod := moduleFactory(s)
export := runtime.NewObject()
export := rt.NewObject()
mod.Export(export) mod.Export(export)
runtime.Set(mod.Name(), export)
rt.Set(mod.Name(), export)
modules = append(modules, mod) modules = append(modules, mod)
} }
s.runtime = runtime for _, mod := range modules {
initMod, ok := mod.(InitializableModule)
if !ok {
continue
}
logger.Debug(context.Background(), "initializing module", logger.F("module", initMod.Name()))
if err := initMod.OnInit(rt); err != nil {
return errors.WithStack(err)
}
}
s.modules = modules s.modules = modules
return nil
} }
func NewServer(factories ...ServerModuleFactory) *Server { func NewServer(factories ...ServerModuleFactory) *Server {
server := &Server{ server := &Server{
factories: factories,
loop: eventloop.NewEventLoop( loop: eventloop.NewEventLoop(
eventloop.EnableConsole(false), eventloop.EnableConsole(false),
), ),
} }
server.initModules(factories...)
return server return server
} }

View File

@ -13,5 +13,5 @@ type ServerModule interface {
type InitializableModule interface { type InitializableModule interface {
ServerModule ServerModule
OnInit() error OnInit(rt *goja.Runtime) error
} }

View File

@ -22,13 +22,13 @@ func (b *Bus) Subscribe(ctx context.Context, ns bus.MessageNamespace) (<-chan bu
) )
dispatchers := b.getDispatchers(ns) dispatchers := b.getDispatchers(ns)
d := newEventDispatcher(b.opt.BufferSize) disp := newEventDispatcher(b.opt.BufferSize)
go d.Run() go disp.Run(ctx)
dispatchers.Add(d) dispatchers.Add(disp)
return d.Out(), nil return disp.Out(), nil
} }
func (b *Bus) Unsubscribe(ctx context.Context, ns bus.MessageNamespace, ch <-chan bus.Message) { func (b *Bus) Unsubscribe(ctx context.Context, ns bus.MessageNamespace, ch <-chan bus.Message) {
@ -52,6 +52,12 @@ func (b *Bus) Publish(ctx context.Context, msg bus.Message) error {
) )
for _, d := range dispatchersList { for _, d := range dispatchersList {
if d.Closed() {
dispatchers.Remove(d)
continue
}
if err := d.In(msg); err != nil { if err := d.In(msg); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }

View File

@ -1,9 +1,13 @@
package memory package memory
import ( import (
"context"
"sync" "sync"
"time"
"forge.cadoles.com/arcad/edge/pkg/bus" "forge.cadoles.com/arcad/edge/pkg/bus"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
) )
type eventDispatcherSet struct { type eventDispatcherSet struct {
@ -18,13 +22,21 @@ func (s *eventDispatcherSet) Add(d *eventDispatcher) {
s.items[d] = struct{}{} s.items[d] = struct{}{}
} }
func (s *eventDispatcherSet) Remove(d *eventDispatcher) {
s.mutex.Lock()
defer s.mutex.Unlock()
d.close()
delete(s.items, d)
}
func (s *eventDispatcherSet) RemoveByOutChannel(out <-chan bus.Message) { func (s *eventDispatcherSet) RemoveByOutChannel(out <-chan bus.Message) {
s.mutex.Lock() s.mutex.Lock()
defer s.mutex.Unlock() defer s.mutex.Unlock()
for d := range s.items { for d := range s.items {
if d.IsOut(out) { if d.IsOut(out) {
d.Close() d.close()
delete(s.items, d) delete(s.items, d)
} }
} }
@ -56,10 +68,21 @@ type eventDispatcher struct {
closed bool closed bool
} }
func (d *eventDispatcher) Closed() bool {
d.mutex.RLock()
defer d.mutex.RUnlock()
return d.closed
}
func (d *eventDispatcher) Close() { func (d *eventDispatcher) Close() {
d.mutex.Lock() d.mutex.Lock()
defer d.mutex.Unlock() defer d.mutex.Unlock()
d.close()
}
func (d *eventDispatcher) close() {
d.closed = true d.closed = true
close(d.in) close(d.in)
} }
@ -85,16 +108,52 @@ func (d *eventDispatcher) IsOut(out <-chan bus.Message) bool {
return d.out == out return d.out == out
} }
func (d *eventDispatcher) Run() { func (d *eventDispatcher) Run(ctx context.Context) {
defer func() {
for {
logger.Debug(ctx, "closing dispatcher, flushing out incoming messages")
close(d.out)
// Flush all incoming messages
for {
_, ok := <-d.in
if !ok {
return
}
}
}
}()
for { for {
msg, ok := <-d.in msg, ok := <-d.in
if !ok { if !ok {
close(d.out)
return return
} }
d.out <- msg timeout := time.After(time.Second)
select {
case d.out <- msg:
case <-timeout:
logger.Error(
ctx,
"out message channel timeout",
logger.F("message", msg),
)
return
case <-ctx.Done():
logger.Error(
ctx,
"message subscription context canceled",
logger.F("message", msg),
logger.E(errors.WithStack(ctx.Err())),
)
return
}
} }
} }

View File

@ -97,6 +97,10 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
bus: opts.Bus, bus: opts.Bus,
} }
for _, middleware := range opts.HTTPMiddlewares {
router.Use(middleware)
}
router.Route("/edge", func(r chi.Router) { router.Route("/edge", func(r chi.Router) {
r.Route("/sdk", func(r chi.Router) { r.Route("/sdk", func(r chi.Router) {
r.Get("/client.js", handler.handleSDKClient) r.Get("/client.js", handler.handleSDKClient)

View File

@ -18,6 +18,7 @@ type HandlerOptions struct {
UploadMaxFileSize int64 UploadMaxFileSize int64
HTTPClient *http.Client HTTPClient *http.Client
HTTPMounts []func(r chi.Router) HTTPMounts []func(r chi.Router)
HTTPMiddlewares []func(next http.Handler) http.Handler
} }
func defaultHandlerOptions() *HandlerOptions { func defaultHandlerOptions() *HandlerOptions {
@ -35,6 +36,7 @@ func defaultHandlerOptions() *HandlerOptions {
Timeout: time.Second * 30, Timeout: time.Second * 30,
}, },
HTTPMounts: make([]func(r chi.Router), 0), HTTPMounts: make([]func(r chi.Router), 0),
HTTPMiddlewares: make([]func(http.Handler) http.Handler, 0),
} }
} }
@ -75,3 +77,9 @@ func WithHTTPMounts(mounts ...func(r chi.Router)) HandlerOptionFunc {
opts.HTTPMounts = mounts opts.HTTPMounts = mounts
} }
} }
func WithHTTPMiddlewares(middlewares ...func(http.Handler) http.Handler) HandlerOptionFunc {
return func(opts *HandlerOptions) {
opts.HTTPMiddlewares = middlewares
}
}

View File

@ -73,12 +73,7 @@ func (h *Handler) serveAppURL(w http.ResponseWriter, r *http.Request) {
from := req.From from := req.From
if from == "" { if from == "" {
host, _, err := net.SplitHostPort(r.RemoteAddr) from = retrieveRemoteAddr(r)
if err != nil {
logger.Warn(ctx, "could not split remote address", logger.E(errors.WithStack(err)))
} else {
from = host
}
} }
url, err := h.repo.GetURL(ctx, appID, from) url, err := h.repo.GetURL(ctx, appID, from)
@ -110,3 +105,12 @@ func Mount(repository Repository) MountFunc {
r.Post("/api/v1/apps/{appID}/url", handler.serveAppURL) r.Post("/api/v1/apps/{appID}/url", handler.serveAppURL)
} }
} }
func retrieveRemoteAddr(r *http.Request) string {
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
host = r.RemoteAddr
}
return host
}

View File

@ -9,6 +9,7 @@ import (
"forge.cadoles.com/arcad/edge/pkg/module/auth" "forge.cadoles.com/arcad/edge/pkg/module/auth"
"forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd" "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd"
"forge.cadoles.com/arcad/edge/pkg/module/auth/jwt"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
"github.com/lestrrat-go/jwx/v2/jwa" "github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jwk"
@ -112,7 +113,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
account.Claims[auth.ClaimIssuer] = "local" account.Claims[auth.ClaimIssuer] = "local"
token, err := generateSignedToken(h.algo, h.key, account.Claims) token, err := jwt.GenerateSignedToken(h.algo, h.key, account.Claims)
if err != nil { if err != nil {
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err))) logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)

View File

@ -30,7 +30,7 @@ func WithJWT(getKeySet GetKeySetFunc) OptionFunc {
} }
} }
func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) { func FindRawToken(r *http.Request) (string, error) {
authorization := r.Header.Get("Authorization") authorization := r.Header.Get("Authorization")
// Retrieve token from Authorization header // Retrieve token from Authorization header
@ -44,7 +44,7 @@ func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) {
if rawToken == "" { if rawToken == "" {
cookie, err := r.Cookie(CookieName) cookie, err := r.Cookie(CookieName)
if err != nil && !errors.Is(err, http.ErrNoCookie) { if err != nil && !errors.Is(err, http.ErrNoCookie) {
return nil, errors.WithStack(err) return "", errors.WithStack(err)
} }
if cookie != nil { if cookie != nil {
@ -53,7 +53,16 @@ func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) {
} }
if rawToken == "" { if rawToken == "" {
return nil, errors.WithStack(ErrUnauthenticated) return "", errors.WithStack(ErrUnauthenticated)
}
return rawToken, nil
}
func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) {
rawToken, err := FindRawToken(r)
if err != nil {
return nil, errors.WithStack(err)
} }
keySet, err := getKeySet() keySet, err := getKeySet()

View File

@ -1,4 +1,4 @@
package http package jwt
import ( import (
"time" "time"
@ -9,7 +9,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
func generateSignedToken(algo jwa.KeyAlgorithm, key jwk.Key, claims map[string]any) ([]byte, error) { func GenerateSignedToken(algo jwa.KeyAlgorithm, key jwk.Key, claims map[string]any) ([]byte, error) {
token := jwt.New() token := jwt.New()
if err := token.Set(jwt.NotBeforeKey, time.Now()); err != nil { if err := token.Set(jwt.NotBeforeKey, time.Now()); err != nil {

View File

@ -0,0 +1,113 @@
package middleware
import (
"crypto/rand"
"fmt"
"math/big"
"net/http"
"time"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
"forge.cadoles.com/arcad/edge/pkg/module/auth/jwt"
"github.com/google/uuid"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
const AnonIssuer = "anon"
func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOptionFunc) func(next http.Handler) http.Handler {
opts := defaultAnonymousUserOptions()
for _, fn := range funcs {
fn(opts)
}
return func(next http.Handler) http.Handler {
handler := func(w http.ResponseWriter, r *http.Request) {
rawToken, err := auth.FindRawToken(r)
// If request already has a raw token, we do nothing
if rawToken != "" && err == nil {
next.ServeHTTP(w, r)
return
}
ctx := r.Context()
uuid, err := uuid.NewUUID()
if err != nil {
logger.Error(ctx, "could not generate uuid for anonymous user", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
subject := fmt.Sprintf("%s-%s", AnonIssuer, uuid.String())
preferredUsername, err := generateRandomPreferredUsername(8)
if err != nil {
logger.Error(ctx, "could not generate preferred username for anonymous user", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
claims := map[string]any{
auth.ClaimSubject: subject,
auth.ClaimIssuer: AnonIssuer,
auth.ClaimPreferredUsername: preferredUsername,
auth.ClaimEdgeRole: opts.Role,
auth.ClaimEdgeEntrypoint: opts.Entrypoint,
auth.ClaimEdgeTenant: opts.Tenant,
}
token, err := jwt.GenerateSignedToken(algo, key, claims)
if err != nil {
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
cookieDomain, err := opts.GetCookieDomain(r)
if err != nil {
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
cookie := http.Cookie{
Name: auth.CookieName,
Value: string(token),
Domain: cookieDomain,
HttpOnly: false,
Expires: time.Now().Add(opts.CookieDuration),
Path: "/",
}
http.SetCookie(w, &cookie)
next.ServeHTTP(w, r)
}
return http.HandlerFunc(handler)
}
}
func generateRandomPreferredUsername(size int) (string, error) {
var letters = []rune("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
max := big.NewInt(int64(len(letters)))
b := make([]rune, size)
for i := range b {
idx, err := rand.Int(rand.Reader, max)
if err != nil {
return "", errors.WithStack(err)
}
b[i] = letters[idx.Int64()]
}
return fmt.Sprintf("Anon %s", string(b)), nil
}

View File

@ -0,0 +1,57 @@
package middleware
import (
"net/http"
"time"
)
type GetCookieDomainFunc func(r *http.Request) (string, error)
func defaultGetCookieDomain(r *http.Request) (string, error) {
return "", nil
}
type AnonymousUserOptions struct {
GetCookieDomain GetCookieDomainFunc
CookieDuration time.Duration
Tenant string
Entrypoint string
Role string
}
type AnonymousUserOptionFunc func(*AnonymousUserOptions)
func defaultAnonymousUserOptions() *AnonymousUserOptions {
return &AnonymousUserOptions{
GetCookieDomain: defaultGetCookieDomain,
CookieDuration: 24 * time.Hour,
Tenant: "",
Entrypoint: "",
Role: "",
}
}
func WithCookieOptions(getCookieDomain GetCookieDomainFunc, duration time.Duration) AnonymousUserOptionFunc {
return func(opts *AnonymousUserOptions) {
opts.GetCookieDomain = getCookieDomain
opts.CookieDuration = duration
}
}
func WithTenant(tenant string) AnonymousUserOptionFunc {
return func(opts *AnonymousUserOptions) {
opts.Tenant = tenant
}
}
func WithEntrypoint(entrypoint string) AnonymousUserOptionFunc {
return func(opts *AnonymousUserOptions) {
opts.Entrypoint = entrypoint
}
}
func WithRole(role string) AnonymousUserOptionFunc {
return func(opts *AnonymousUserOptions) {
opts.Role = role
}
}

View File

@ -53,6 +53,10 @@ func (m *Module) Export(export *goja.Object) {
if err := export.Set("CLAIM_PREFERRED_USERNAME", ClaimPreferredUsername); err != nil { if err := export.Set("CLAIM_PREFERRED_USERNAME", ClaimPreferredUsername); err != nil {
panic(errors.Wrap(err, "could not set 'CLAIM_PREFERRED_USERNAME' property")) panic(errors.Wrap(err, "could not set 'CLAIM_PREFERRED_USERNAME' property"))
} }
if err := export.Set("CLAIM_ISSUER", ClaimIssuer); err != nil {
panic(errors.Wrap(err, "could not set 'CLAIM_ISSUER' property"))
}
} }
func (m *Module) getClaim(call goja.FunctionCall, rt *goja.Runtime) goja.Value { func (m *Module) getClaim(call goja.FunctionCall, rt *goja.Runtime) goja.Value {

View File

@ -1,14 +1,14 @@
package blob package blob
import ( import (
"io/ioutil" "os"
"testing" "testing"
"cdr.dev/slog" "cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app" "forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus/memory" "forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/arcad/edge/pkg/module" "forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite" "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"github.com/pkg/errors" "github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger" "gitlab.com/wpetit/goweb/logger"
) )
@ -27,7 +27,7 @@ func TestBlobModule(t *testing.T) {
ModuleFactory(bus, store), ModuleFactory(bus, store),
) )
data, err := ioutil.ReadFile("testdata/blob.js") data, err := os.ReadFile("testdata/blob.js")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -15,10 +15,7 @@ const (
defaultTimeout = 30 * time.Second defaultTimeout = 30 * time.Second
) )
type Module struct { type Module struct{}
ctx context.Context
server *app.Server
}
func (m *Module) Name() string { func (m *Module) Name() string {
return "cast" return "cast"
@ -54,7 +51,7 @@ func (m *Module) refreshDevices(call goja.FunctionCall, rt *goja.Runtime) goja.V
panic(rt.ToValue(errors.WithStack(err))) panic(rt.ToValue(errors.WithStack(err)))
} }
promise := m.server.NewPromise() promise := app.NewPromiseProxyFrom(rt)
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -102,7 +99,7 @@ func (m *Module) loadUrl(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
panic(rt.ToValue(errors.WithStack(err))) panic(rt.ToValue(errors.WithStack(err)))
} }
promise := m.server.NewPromise() promise := app.NewPromiseProxyFrom(rt)
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -121,7 +118,7 @@ func (m *Module) loadUrl(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
promise.Resolve(nil) promise.Resolve(nil)
}() }()
return m.server.ToValue(promise) return rt.ToValue(promise)
} }
func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value { func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
@ -137,7 +134,7 @@ func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
panic(rt.ToValue(errors.WithStack(err))) panic(rt.ToValue(errors.WithStack(err)))
} }
promise := m.server.NewPromise() promise := app.NewPromiseProxyFrom(rt)
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -156,7 +153,7 @@ func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
promise.Resolve(nil) promise.Resolve(nil)
}() }()
return m.server.ToValue(promise) return rt.ToValue(promise)
} }
func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value { func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
@ -172,7 +169,7 @@ func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value
panic(rt.ToValue(errors.WithStack(err))) panic(rt.ToValue(errors.WithStack(err)))
} }
promise := m.server.NewPromise() promise := app.NewPromiseProxyFrom(rt)
go func() { go func() {
ctx, cancel := context.WithTimeout(context.Background(), timeout) ctx, cancel := context.WithTimeout(context.Background(), timeout)
@ -191,7 +188,7 @@ func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value
promise.Resolve(status) promise.Resolve(status)
}() }()
return m.server.ToValue(promise) return rt.ToValue(promise)
} }
func (m *Module) parseTimeout(rawTimeout string) (time.Duration, error) { func (m *Module) parseTimeout(rawTimeout string) (time.Duration, error) {
@ -214,8 +211,6 @@ func (m *Module) parseTimeout(rawTimeout string) (time.Duration, error) {
func CastModuleFactory() app.ServerModuleFactory { func CastModuleFactory() app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule { return func(server *app.Server) app.ServerModule {
return &Module{ return &Module{}
server: server,
}
} }
} }

View File

@ -85,7 +85,7 @@ func TestCastModuleRefreshDevices(t *testing.T) {
t.Error(errors.WithStack(err)) t.Error(errors.WithStack(err))
} }
promise, ok := server.IsPromise(result) promise, ok := app.IsPromise(result)
if !ok { if !ok {
t.Fatal("expected promise") t.Fatal("expected promise")
} }

View File

@ -9,9 +9,7 @@ import (
"gitlab.com/wpetit/goweb/logger" "gitlab.com/wpetit/goweb/logger"
) )
type LifecycleModule struct { type LifecycleModule struct{}
server *app.Server
}
func (m *LifecycleModule) Name() string { func (m *LifecycleModule) Name() string {
return "lifecycle" return "lifecycle"
@ -20,25 +18,37 @@ func (m *LifecycleModule) Name() string {
func (m *LifecycleModule) Export(export *goja.Object) { func (m *LifecycleModule) Export(export *goja.Object) {
} }
func (m *LifecycleModule) OnInit() error { func (m *LifecycleModule) OnInit(rt *goja.Runtime) (err error) {
if _, err := m.server.ExecFuncByName(context.Background(), "onInit"); err != nil { call, ok := goja.AssertFunction(rt.Get("onInit"))
if errors.Is(err, app.ErrFuncDoesNotExist) { if !ok {
logger.Warn(context.Background(), "could not find onInit() function", logger.E(errors.WithStack(err))) logger.Warn(context.Background(), "could not find onInit() function")
return nil return nil
} }
return errors.WithStack(err) defer func() {
if recovered := recover(); recovered != nil {
revoveredErr, ok := recovered.(error)
if ok {
logger.Error(context.Background(), "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
err = errors.WithStack(app.ErUnknownError)
return
} }
panic(recovered)
}
}()
call(nil)
return nil return nil
} }
func LifecycleModuleFactory() app.ServerModuleFactory { func LifecycleModuleFactory() app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule { return func(server *app.Server) app.ServerModule {
module := &LifecycleModule{ module := &LifecycleModule{}
server: server,
}
return module return module
} }

View File

@ -51,6 +51,12 @@ func (m *RPCModule) Export(export *goja.Object) {
} }
} }
func (m *RPCModule) OnInit(rt *goja.Runtime) error {
go m.handleMessages()
return nil
}
func (m *RPCModule) register(call goja.FunctionCall, rt *goja.Runtime) goja.Value { func (m *RPCModule) register(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
fnName := util.AssertString(call.Argument(0), rt) fnName := util.AssertString(call.Argument(0), rt)
@ -117,16 +123,21 @@ func (m *RPCModule) handleMessages() {
} }
for msg := range clientMessages { for msg := range clientMessages {
go m.handleMessage(ctx, msg, sendRes)
}
}
func (m *RPCModule) handleMessage(ctx context.Context, msg bus.Message, sendRes func(ctx context.Context, req *RPCRequest, result goja.Value)) {
clientMessage, ok := msg.(*ClientMessage) clientMessage, ok := msg.(*ClientMessage)
if !ok { if !ok {
logger.Warn(ctx, "unexpected bus message", logger.F("message", msg)) logger.Warn(ctx, "unexpected bus message", logger.F("message", msg))
continue return
} }
ok, req := m.isRPCRequest(clientMessage) ok, req := m.isRPCRequest(clientMessage)
if !ok { if !ok {
continue return
} }
logger.Debug(ctx, "received rpc request", logger.F("request", req)) logger.Debug(ctx, "received rpc request", logger.F("request", req))
@ -143,7 +154,7 @@ func (m *RPCModule) handleMessages() {
) )
} }
continue return
} }
callable, ok := rawCallable.(goja.Callable) callable, ok := rawCallable.(goja.Callable)
@ -158,7 +169,7 @@ func (m *RPCModule) handleMessages() {
) )
} }
continue return
} }
result, err := m.server.Exec(clientMessage.Context, callable, clientMessage.Context, req.Params) result, err := m.server.Exec(clientMessage.Context, callable, clientMessage.Context, req.Params)
@ -178,10 +189,10 @@ func (m *RPCModule) handleMessages() {
) )
} }
continue return
} }
promise, ok := m.server.IsPromise(result) promise, ok := app.IsPromise(result)
if ok { if ok {
go func(ctx context.Context, req *RPCRequest, promise *goja.Promise) { go func(ctx context.Context, req *RPCRequest, promise *goja.Promise) {
result := m.server.WaitForPromise(promise) result := m.server.WaitForPromise(promise)
@ -190,7 +201,6 @@ func (m *RPCModule) handleMessages() {
} else { } else {
sendRes(clientMessage.Context, req, result) sendRes(clientMessage.Context, req, result)
} }
}
} }
func (m *RPCModule) sendErrorResponse(ctx context.Context, req *RPCRequest, err error) error { func (m *RPCModule) sendErrorResponse(ctx context.Context, req *RPCRequest, err error) error {
@ -263,8 +273,8 @@ func RPCModuleFactory(bus bus.Bus) app.ServerModuleFactory {
bus: bus, bus: bus,
} }
go mod.handleMessages()
return mod return mod
} }
} }
var _ app.InitializableModule = &RPCModule{}

342
pkg/module/share/module.go Normal file
View File

@ -0,0 +1,342 @@
package share
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module/util"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/dop251/goja"
"github.com/pkg/errors"
)
const (
AnyType share.ValueType = "*"
AnyName string = "*"
)
type Module struct {
appID app.ID
store share.Store
}
func (m *Module) Name() string {
return "share"
}
func (m *Module) Export(export *goja.Object) {
if err := export.Set("upsertResource", m.upsertResource); err != nil {
panic(errors.Wrap(err, "could not set 'upsertResource' function"))
}
if err := export.Set("findResources", m.findResources); err != nil {
panic(errors.Wrap(err, "could not set 'findResources' function"))
}
if err := export.Set("deleteAttributes", m.deleteAttributes); err != nil {
panic(errors.Wrap(err, "could not set 'deleteAttributes' function"))
}
if err := export.Set("deleteResource", m.deleteResource); err != nil {
panic(errors.Wrap(err, "could not set 'deleteResource' function"))
}
if err := export.Set("ANY_TYPE", AnyType); err != nil {
panic(errors.Wrap(err, "could not set 'ANY_TYPE' property"))
}
if err := export.Set("ANY_NAME", AnyName); err != nil {
panic(errors.Wrap(err, "could not set 'ANY_NAME' property"))
}
if err := export.Set("TYPE_TEXT", share.TypeText); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_TEXT' property"))
}
if err := export.Set("TYPE_NUMBER", share.TypeNumber); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_NUMBER' property"))
}
if err := export.Set("TYPE_BOOL", share.TypeBool); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_BOOL' property"))
}
if err := export.Set("TYPE_PATH", share.TypePath); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_PATH' property"))
}
}
func (m *Module) upsertResource(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
resourceID := assertResourceID(call.Argument(1), rt)
var attributes []share.Attribute
if len(call.Arguments) > 2 {
attributes = assertAttributes(call.Arguments[2:], rt)
} else {
attributes = make([]share.Attribute, 0)
}
for _, attr := range attributes {
if err := share.AssertType(attr.Value(), attr.Type()); err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
}
resource, err := m.store.UpdateAttributes(ctx, m.appID, resourceID, attributes...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(toGojaResource(resource))
}
func (m *Module) deleteAttributes(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
resourceID := assertResourceID(call.Argument(1), rt)
var names []string
if len(call.Arguments) > 2 {
names = assertStrings(call.Arguments[2:], rt)
} else {
names = make([]string, 0)
}
err := m.store.DeleteAttributes(ctx, m.appID, resourceID, names...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return nil
}
func (m *Module) findResources(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
funcs := make([]share.FindResourcesOptionFunc, 0)
if len(call.Arguments) > 1 {
name := util.AssertString(call.Argument(1), rt)
if name != AnyName {
funcs = append(funcs, share.WithName(name))
}
}
if len(call.Arguments) > 2 {
valueType := assertValueType(call.Argument(2), rt)
if valueType != AnyType {
funcs = append(funcs, share.WithType(valueType))
}
}
resources, err := m.store.FindResources(ctx, funcs...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(toGojaResources(resources))
}
func (m *Module) deleteResource(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
resourceID := assertResourceID(call.Argument(1), rt)
err := m.store.DeleteResource(ctx, m.appID, resourceID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return nil
}
func ModuleFactory(appID app.ID, store share.Store) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
return &Module{
appID: appID,
store: store,
}
}
}
func assertResourceID(v goja.Value, r *goja.Runtime) share.ResourceID {
value := v.Export()
switch typ := value.(type) {
case string:
return share.ResourceID(typ)
case share.ResourceID:
return typ
default:
panic(r.ToValue(errors.Errorf("expected value to be a string or ResourceID, got '%T'", value)))
}
}
func assertAttributes(values []goja.Value, r *goja.Runtime) []share.Attribute {
attributes := make([]share.Attribute, len(values))
for idx, val := range values {
export := val.Export()
rawAttr, ok := export.(map[string]any)
if !ok {
panic(r.ToValue(errors.Errorf("unexpected attribute value, got '%v'", export)))
}
rawName, exists := rawAttr["name"]
if !exists {
panic(r.ToValue(errors.Errorf("could not find 'name' property on attribute '%v'", export)))
}
name, ok := rawName.(string)
if !ok {
panic(r.ToValue(errors.Errorf("unexpected value for attribute property 'name': expected 'string', got '%T'", rawName)))
}
rawType, exists := rawAttr["type"]
if !exists {
panic(r.ToValue(errors.Errorf("could not find 'type' property on attribute '%v'", export)))
}
var valueType share.ValueType
switch typ := rawType.(type) {
case share.ValueType:
valueType = typ
case string:
valueType = share.ValueType(typ)
default:
panic(r.ToValue(errors.Errorf("unexpected value for attribute property 'type': expected 'string' or 'ValueType', got '%T'", rawType)))
}
value, exists := rawAttr["value"]
if !exists {
panic(r.ToValue(errors.Errorf("could not find 'value' property on attribute '%v'", export)))
}
attributes[idx] = share.NewBaseAttribute(
name,
valueType,
value,
)
}
return attributes
}
func assertStrings(values []goja.Value, r *goja.Runtime) []string {
strings := make([]string, len(values))
for idx, v := range values {
strings[idx] = util.AssertString(v, r)
}
return strings
}
func assertValueType(v goja.Value, r *goja.Runtime) share.ValueType {
value := v.Export()
switch typ := value.(type) {
case string:
return share.ValueType(typ)
case share.ValueType:
return typ
default:
panic(r.ToValue(errors.Errorf("expected value to be a string or ValueType, got '%T'", value)))
}
}
type gojaResource struct {
ID share.ResourceID `goja:"id" json:"id"`
Origin app.ID `goja:"origin" json:"origin"`
Attributes []*gojaAttribute `goja:"attributes" json:"attributes"`
}
func (r *gojaResource) Has(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
name := util.AssertString(call.Argument(0), rt)
valueType := assertValueType(call.Argument(1), rt)
hasAttr := share.HasAttribute(toResource(r), name, valueType)
return rt.ToValue(hasAttr)
}
func (r *gojaResource) Get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
name := util.AssertString(call.Argument(0), rt)
valueType := assertValueType(call.Argument(1), rt)
var defaultValue any
if len(call.Arguments) > 2 {
defaultValue = call.Argument(2).Export()
}
attr := share.GetAttribute(toResource(r), name, valueType)
if attr == nil {
return rt.ToValue(defaultValue)
}
return rt.ToValue(attr.Value())
}
type gojaAttribute struct {
Name string `goja:"name" json:"name"`
Type share.ValueType `goja:"type" json:"type"`
Value any `goja:"value" json:"value"`
CreatedAt time.Time `goja:"createdAt" json:"createdAt"`
UpdatedAt time.Time `goja:"updatedAt" json:"updatedAt"`
}
func toGojaResource(res share.Resource) *gojaResource {
attributes := make([]*gojaAttribute, len(res.Attributes()))
for idx, attr := range res.Attributes() {
attributes[idx] = &gojaAttribute{
Name: attr.Name(),
Type: attr.Type(),
Value: attr.Value(),
CreatedAt: attr.CreatedAt(),
UpdatedAt: attr.UpdatedAt(),
}
}
return &gojaResource{
ID: res.ID(),
Origin: res.Origin(),
Attributes: attributes,
}
}
func toGojaResources(resources []share.Resource) []*gojaResource {
gojaResources := make([]*gojaResource, len(resources))
for idx, res := range resources {
gojaResources[idx] = toGojaResource(res)
}
return gojaResources
}
func toResource(res *gojaResource) share.Resource {
return share.NewBaseResource(
res.Origin,
res.ID,
toAttributes(res.Attributes)...,
)
}
func toAttributes(gojaAttributes []*gojaAttribute) []share.Attribute {
attributes := make([]share.Attribute, len(gojaAttributes))
for idx, gojaAttr := range gojaAttributes {
attr := share.NewBaseAttribute(
gojaAttr.Name,
gojaAttr.Type,
gojaAttr.Value,
)
attr.SetCreatedAt(gojaAttr.CreatedAt)
attr.SetUpdatedAt(gojaAttr.UpdatedAt)
attributes[idx] = attr
}
return attributes
}

View File

@ -0,0 +1,49 @@
package share
import (
"context"
"os"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
)
func TestModule(t *testing.T) {
logger.SetLevel(logger.LevelDebug)
store, err := driver.NewShareStore("sqlite://testdata/test_share_module.sqlite")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
ModuleFactory("test.app.edge", store),
)
data, err := os.ReadFile("testdata/share.js")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Load("testdata/share.js", string(data)); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Start(); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if _, err := server.ExecFuncByName(context.Background(), "testModule"); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
server.Stop()
}

82
pkg/module/share/testdata/share.js vendored Normal file
View File

@ -0,0 +1,82 @@
function testModule() {
var ctx = context.new();
var resourceId = "my-first-res";
var attributes = [
{ name: "my_text", type: share.TYPE_TEXT, value: "my_text" },
{ name: "my_number", type: share.TYPE_NUMBER, value: 5 },
{ name: "my_path", type: share.TYPE_PATH, value: "/my/path" },
{ name: "my_bool", type: share.TYPE_BOOL, value: true },
]
// Create resource with attributes
var resource = share.upsertResource(
ctx, resourceId,
attributes[0],
attributes[1],
attributes[2],
attributes[3]
);
if (resource.id != resourceId) {
throw new Error("resource.id: expected '"+resourceId+"', got '"+resource.id+"'")
}
if (resource.origin != "test.app.edge") {
throw new Error("resource.origin: expected 'test.app.edge', got '"+resource.origin+"'")
}
if (resource.attributes.length != 4) {
throw new Error("resource.attributes.length: expected '1', got '"+resource.attributes.length+"'")
}
for(var attr, i = 0;( attr = attributes[i] ); i++) {
var exists = resource.has(attr.name, attr.type);
if (!exists) {
throw new Error("resource.has('"+attr.name+"'): expected 'true', got '"+hasAttr+"'")
}
var value = resource.get(attr.name, attr.type);
if (value != attr.value) {
throw new Error("value: expected '"+attr.value+"', got '"+value+"'")
}
}
// Test acces of unexistant attribute
var unexistantAttr = "unexistant_attr"
var exists = resource.has(unexistantAttr, share.TYPE_TEXT);
if (exists) {
throw new Error("attr '"+unexistantAttr+"' should not exist")
}
var expected = "foo"
var value = resource.get(unexistantAttr, share.TYPE_TEXT, expected);
if (value != expected) {
throw new Error("resource.get('"+attr.name+"', share.TYPE_TEXT, '"+expected+"'): expected '"+expected+"', got '"+value+"'")
}
// Search resources
// With any attribute
var results = share.findResources(ctx, share.ANY_NAME, share.ANY_TYPE);
if (results.length != 1) {
throw new Error("results.length: expected '1', got '"+results.length+"'")
}
// With an unexistant attribute
var results = share.findResources(ctx, unexistantAttr, share.ANY_TYPE);
if (results.length != 0) {
throw new Error("results.length: expected '0', got '"+results.length+"'")
}
// With a wrong type
var results = share.findResources(ctx, "my_text", share.TYPE_NUMBER);
if (results.length != 0) {
throw new Error("results.length: expected '0', got '"+results.length+"'")
}
}

View File

@ -2,12 +2,12 @@ package store
import ( import (
"context" "context"
"io/ioutil" "os"
"testing" "testing"
"forge.cadoles.com/arcad/edge/pkg/app" "forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module" "forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite" "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"github.com/pkg/errors" "github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger" "gitlab.com/wpetit/goweb/logger"
) )
@ -22,7 +22,7 @@ func TestStoreModule(t *testing.T) {
ModuleFactory(store), ModuleFactory(store),
) )
data, err := ioutil.ReadFile("testdata/store.js") data, err := os.ReadFile("testdata/store.js")
if err != nil { if err != nil {
t.Fatalf("%+v", errors.WithStack(err)) t.Fatalf("%+v", errors.WithStack(err))
} }

View File

@ -3,6 +3,7 @@ package util
import ( import (
"context" "context"
"forge.cadoles.com/arcad/edge/pkg/app"
"github.com/dop251/goja" "github.com/dop251/goja"
"github.com/pkg/errors" "github.com/pkg/errors"
) )
@ -26,3 +27,15 @@ func AssertObject(v goja.Value, r *goja.Runtime) map[string]any {
func AssertString(v goja.Value, r *goja.Runtime) string { func AssertString(v goja.Value, r *goja.Runtime) string {
return AssertType[string](v, r) return AssertType[string](v, r)
} }
func AssertAppID(v goja.Value, r *goja.Runtime) app.ID {
value := v.Export()
switch typ := value.(type) {
case string:
return app.ID(typ)
case app.ID:
return typ
default:
panic(r.ToValue(errors.Errorf("expected value to be a string or app.ID, got '%T'", value)))
}
}

View File

@ -92,7 +92,7 @@ var Edge=(()=>{var K3=Object.create;var Mi=Object.defineProperty,Y3=Object.defin
</edge-menu-item> </edge-menu-item>
`}_canAccess(t){var a,o;let i=((a=this._profile)==null?void 0:a.edge_role)||"visitor",n=((o=t.metadata)==null?void 0:o.minimumRole)||"visitor";return sb[i]>=sb[n]}_renderProfile(){let t=this._profile;return re` `}_canAccess(t){var a,o;let i=((a=this._profile)==null?void 0:a.edge_role)||"visitor",n=((o=t.metadata)==null?void 0:o.minimumRole)||"visitor";return sb[i]>=sb[n]}_renderProfile(){let t=this._profile;return re`
<edge-menu-item name='profile' label="${(t==null?void 0:t.preferred_username)||"Profile"}" icon-url='${Zm}'> <edge-menu-item name='profile' label="${(t==null?void 0:t.preferred_username)||"Profile"}" icon-url='${Zm}'>
${t?re`<edge-menu-sub-item name='login' label='Logout' icon-url='${nb}' link-url='/edge/auth/logout'></edge-menu-sub-item>`:re`<edge-menu-sub-item name='login' label='Login' icon-url='${tb}' link-url='/edge/auth/login'></edge-menu-sub-item>`} ${t&&t.iss!="anon"?re`<edge-menu-sub-item name='login' label='Logout' icon-url='${nb}' link-url='/edge/auth/logout'></edge-menu-sub-item>`:re`<edge-menu-sub-item name='login' label='Login' icon-url='${tb}' link-url='/edge/auth/login'></edge-menu-sub-item>`}
</edge-menu-item> </edge-menu-item>
`}_handleMenuItemSelected(t){let i=t.detail.element;i.classList.add("selected"),i.classList.remove("unselected");for(let n,a=0;n=this._menuItems[a];a++)n!==i&&(n.unselect(),n.classList.add("unselected"))}_handleMenuItemUnselected(t){if(t.detail.element.classList.remove("selected"),this.renderRoot.querySelectorAll("edge-menu-item.selected").length===0)for(let a,o=0;a=this._menuItems[o];o++)a.classList.remove("unselected")}};le.styles=Ti` `}_handleMenuItemSelected(t){let i=t.detail.element;i.classList.add("selected"),i.classList.remove("unselected");for(let n,a=0;n=this._menuItems[a];a++)n!==i&&(n.unselect(),n.classList.add("unselected"))}_handleMenuItemUnselected(t){if(t.detail.element.classList.remove("selected"),this.renderRoot.querySelectorAll("edge-menu-item.selected").length===0)for(let a,o=0;a=this._menuItems[o];o++)a.classList.remove("unselected")}};le.styles=Ti`
:host { :host {

File diff suppressed because one or more lines are too long

View File

@ -191,7 +191,7 @@ export class Menu extends LitElement {
return html` return html`
<edge-menu-item name='profile' label="${profile?.preferred_username || 'Profile'}" icon-url='${UserCircleIcon}'> <edge-menu-item name='profile' label="${profile?.preferred_username || 'Profile'}" icon-url='${UserCircleIcon}'>
${ ${
profile ? profile && profile.iss != "anon" ?
html`<edge-menu-sub-item name='login' label='Logout' icon-url='${LogoutIcon}' link-url='/edge/auth/logout'></edge-menu-sub-item>` : html`<edge-menu-sub-item name='login' label='Logout' icon-url='${LogoutIcon}' link-url='/edge/auth/logout'></edge-menu-sub-item>` :
html`<edge-menu-sub-item name='login' label='Login' icon-url='${LoginIcon}' link-url='/edge/auth/login'></edge-menu-sub-item>` html`<edge-menu-sub-item name='login' label='Login' icon-url='${LoginIcon}' link-url='/edge/auth/login'></edge-menu-sub-item>`
} }

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
var blobStoreFactories = make(map[string]BlobStoreFactory, 0)
type BlobStoreFactory func(url *url.URL) (storage.BlobStore, error)
func RegisterBlobStoreFactory(scheme string, factory BlobStoreFactory) {
blobStoreFactories[scheme] = factory
}
func NewBlobStore(dsn string) (storage.BlobStore, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := blobStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
var documentStoreFactories = make(map[string]DocumentStoreFactory, 0)
type DocumentStoreFactory func(url *url.URL) (storage.DocumentStore, error)
func RegisterDocumentStoreFactory(scheme string, factory DocumentStoreFactory) {
documentStoreFactories[scheme] = factory
}
func NewDocumentStore(dsn string) (storage.DocumentStore, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := documentStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -0,0 +1,5 @@
package driver
import "errors"
var ErrSchemeNotRegistered = errors.New("scheme was not registered")

View File

@ -0,0 +1,239 @@
package client
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"github.com/pkg/errors"
)
type BlobBucket struct {
name string
id blob.BucketID
call CallFunc
}
// Size implements storage.BlobBucket
func (b *BlobBucket) Size(ctx context.Context) (int64, error) {
args := blob.GetBucketSizeArgs{
BucketID: b.id,
}
reply := blob.GetBucketSizeReply{}
if err := b.call(ctx, "Service.GetBucketSize", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Size, nil
}
// Name implements storage.BlobBucket
func (b *BlobBucket) Name() string {
return b.name
}
// Close implements storage.BlobBucket
func (b *BlobBucket) Close() error {
args := blob.CloseBucketArgs{
BucketID: b.id,
}
reply := blob.CloseBucketReply{}
if err := b.call(context.Background(), "Service.CloseBucket", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Delete implements storage.BlobBucket
func (b *BlobBucket) Delete(ctx context.Context, id storage.BlobID) error {
args := blob.DeleteBucketArgs{
BucketName: b.name,
}
reply := blob.DeleteBucketReply{}
if err := b.call(context.Background(), "Service.DeleteBucket", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.BlobBucket
func (b *BlobBucket) Get(ctx context.Context, id storage.BlobID) (storage.BlobInfo, error) {
args := blob.GetBlobInfoArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.GetBlobInfoReply{}
if err := b.call(context.Background(), "Service.GetBlobInfo", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.BlobInfo, nil
}
// List implements storage.BlobBucket
func (b *BlobBucket) List(ctx context.Context) ([]storage.BlobInfo, error) {
args := blob.ListBlobInfoArgs{
BucketID: b.id,
}
reply := blob.ListBlobInfoReply{}
if err := b.call(context.Background(), "Service.ListBlobInfo", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.BlobInfos, nil
}
// NewReader implements storage.BlobBucket
func (b *BlobBucket) NewReader(ctx context.Context, id storage.BlobID) (io.ReadSeekCloser, error) {
args := blob.NewBlobReaderArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.NewBlobReaderReply{}
if err := b.call(context.Background(), "Service.NewBlobReader", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return &blobReaderCloser{
readerID: reply.ReaderID,
call: b.call,
}, nil
}
// NewWriter implements storage.BlobBucket
func (b *BlobBucket) NewWriter(ctx context.Context, id storage.BlobID) (io.WriteCloser, error) {
args := blob.NewBlobWriterArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.NewBlobWriterReply{}
if err := b.call(context.Background(), "Service.NewBlobWriter", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return &blobWriterCloser{
blobID: id,
writerID: reply.WriterID,
call: b.call,
}, nil
}
type blobWriterCloser struct {
blobID storage.BlobID
writerID blob.WriterID
call CallFunc
}
// Write implements io.WriteCloser
func (bwc *blobWriterCloser) Write(data []byte) (int, error) {
args := blob.WriteBlobArgs{
WriterID: bwc.writerID,
Data: data,
}
reply := blob.WriteBlobReply{}
if err := bwc.call(context.Background(), "Service.WriteBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Written, nil
}
// Close implements io.WriteCloser
func (bwc *blobWriterCloser) Close() error {
args := blob.CloseWriterArgs{
WriterID: bwc.writerID,
}
reply := blob.CloseBucketReply{}
if err := bwc.call(context.Background(), "Service.CloseWriter", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
type blobReaderCloser struct {
readerID blob.ReaderID
call func(ctx context.Context, serviceMethod string, args any, reply any) error
}
// Read implements io.ReadSeekCloser
func (brc *blobReaderCloser) Read(p []byte) (int, error) {
args := blob.ReadBlobArgs{
ReaderID: brc.readerID,
Length: len(p),
}
reply := blob.ReadBlobReply{}
if err := brc.call(context.Background(), "Service.ReadBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
copy(p, reply.Data)
if reply.EOF {
return reply.Read, io.EOF
}
return reply.Read, nil
}
// Seek implements io.ReadSeekCloser
func (brc *blobReaderCloser) Seek(offset int64, whence int) (int64, error) {
args := blob.SeekBlobArgs{
ReaderID: brc.readerID,
Offset: offset,
Whence: whence,
}
reply := blob.SeekBlobReply{}
if err := brc.call(context.Background(), "Service.SeekBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Read, nil
}
// Close implements io.ReadSeekCloser
func (brc *blobReaderCloser) Close() error {
args := blob.CloseReaderArgs{
ReaderID: brc.readerID,
}
reply := blob.CloseReaderReply{}
if err := brc.call(context.Background(), "Service.CloseReader", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
var (
_ storage.BlobBucket = &BlobBucket{}
_ storage.BlobInfo = &BlobInfo{}
_ io.WriteCloser = &blobWriterCloser{}
_ io.ReadSeekCloser = &blobReaderCloser{}
)

View File

@ -0,0 +1,40 @@
package client
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
type BlobInfo struct {
id storage.BlobID
bucket string
contentType string
modTime time.Time
size int64
}
// Bucket implements storage.BlobInfo
func (i *BlobInfo) Bucket() string {
return i.bucket
}
// ID implements storage.BlobInfo
func (i *BlobInfo) ID() storage.BlobID {
return i.id
}
// ContentType implements storage.BlobInfo
func (i *BlobInfo) ContentType() string {
return i.contentType
}
// ModTime implements storage.BlobInfo
func (i *BlobInfo) ModTime() time.Time {
return i.modTime
}
// Size implements storage.BlobInfo
func (i *BlobInfo) Size() int64 {
return i.size
}

View File

@ -0,0 +1,101 @@
package client
import (
"context"
"net/url"
"github.com/keegancsmith/rpc"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"github.com/pkg/errors"
)
type BlobStore struct {
serverURL *url.URL
}
// DeleteBucket implements storage.BlobStore.
func (s *BlobStore) DeleteBucket(ctx context.Context, name string) error {
args := &blob.DeleteBucketArgs{
BucketName: name,
}
if err := s.call(ctx, "Service.DeleteBucket", args, nil); err != nil {
return errors.WithStack(err)
}
return nil
}
// ListBuckets implements storage.BlobStore.
func (s *BlobStore) ListBuckets(ctx context.Context) ([]string, error) {
args := &blob.ListBucketsArgs{}
reply := blob.ListBucketsReply{}
if err := s.call(ctx, "Service.ListBuckets", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Buckets, nil
}
// OpenBucket implements storage.BlobStore.
func (s *BlobStore) OpenBucket(ctx context.Context, name string) (storage.BlobBucket, error) {
args := &blob.OpenBucketArgs{
BucketName: name,
}
reply := &blob.OpenBucketReply{}
if err := s.call(ctx, "Service.OpenBucket", args, reply); err != nil {
return nil, errors.WithStack(err)
}
return &BlobBucket{
name: name,
id: reply.BucketID,
call: s.call,
}, nil
}
func (s *BlobStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *BlobStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewBlobStore(serverURL *url.URL) *BlobStore {
return &BlobStore{serverURL}
}
var _ storage.BlobStore = &BlobStore{}

View File

@ -0,0 +1,87 @@
package client
import (
"context"
"fmt"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestBlobStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
httpServer, err := startNewBlobStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewBlobStore(serverURL)
testsuite.TestBlobStore(context.Background(), t, store)
}
func BenchmarkBlobStore(t *testing.B) {
logger.SetLevel(logger.LevelError)
httpServer, err := startNewBlobStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewBlobStore(serverURL)
testsuite.BenchmarkBlobStore(t, store)
}
func getSQLiteBlobStore() (*sqlite.BlobStore, error) {
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewBlobStore(dsn)
return store, nil
}
func startNewBlobStoreServer() (*httptest.Server, error) {
store, err := getSQLiteBlobStore()
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewBlobStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1,134 @@
package client
import (
"context"
"net/url"
"github.com/keegancsmith/rpc"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/document"
"forge.cadoles.com/arcad/edge/pkg/storage/filter"
)
type DocumentStore struct {
serverURL *url.URL
}
// Delete implements storage.DocumentStore.
func (s *DocumentStore) Delete(ctx context.Context, collection string, id storage.DocumentID) error {
args := document.DeleteDocumentArgs{
Collection: collection,
DocumentID: id,
}
reply := document.DeleteDocumentReply{}
if err := s.call(ctx, "Service.DeleteDocument", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.DocumentStore.
func (s *DocumentStore) Get(ctx context.Context, collection string, id storage.DocumentID) (storage.Document, error) {
args := document.GetDocumentArgs{
Collection: collection,
DocumentID: id,
}
reply := document.GetDocumentReply{}
if err := s.call(ctx, "Service.GetDocument", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Document, nil
}
// Query implements storage.DocumentStore.
func (s *DocumentStore) Query(ctx context.Context, collection string, filter *filter.Filter, funcs ...storage.QueryOptionFunc) ([]storage.Document, error) {
opts := &storage.QueryOptions{}
for _, fn := range funcs {
fn(opts)
}
args := document.QueryDocumentsArgs{
Collection: collection,
Filter: nil,
Options: opts,
}
if filter != nil {
args.Filter = filter.AsMap()
}
reply := document.QueryDocumentsReply{
Documents: []storage.Document{},
}
if err := s.call(ctx, "Service.QueryDocuments", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Documents, nil
}
// Upsert implements storage.DocumentStore.
func (s *DocumentStore) Upsert(ctx context.Context, collection string, doc storage.Document) (storage.Document, error) {
args := document.UpsertDocumentArgs{
Collection: collection,
Document: doc,
}
reply := document.UpsertDocumentReply{}
if err := s.call(ctx, "Service.UpsertDocument", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Document, nil
}
func (s *DocumentStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *DocumentStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewDocumentStore(url *url.URL) *DocumentStore {
return &DocumentStore{url}
}
var _ storage.DocumentStore = &DocumentStore{}

View File

@ -0,0 +1,67 @@
package client
import (
"context"
"fmt"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestDocumentStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
httpServer, err := startNewDocumentStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewDocumentStore(serverURL)
testsuite.TestDocumentStore(context.Background(), t, store)
}
func getSQLiteDocumentStore() (*sqlite.DocumentStore, error) {
file := "./testdata/documentstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewDocumentStore(dsn)
return store, nil
}
func startNewDocumentStoreServer() (*httptest.Server, error) {
store, err := getSQLiteDocumentStore()
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewDocumentStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1,17 @@
package client
import (
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
func remapShareError(err error) error {
switch errors.Cause(err).Error() {
case share.ErrAttributeRequired.Error():
return share.ErrAttributeRequired
case share.ErrNotFound.Error():
return share.ErrNotFound
default:
return err
}
}

View File

@ -0,0 +1,9 @@
package client
import (
"context"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
)
type CallFunc func(ctx context.Context, serviceMethod string, args any, reply any) error

View File

@ -0,0 +1,150 @@
package client
import (
"context"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/app"
server "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/keegancsmith/rpc"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type ShareStore struct {
serverURL *url.URL
}
// DeleteAttributes implements share.Store.
func (s *ShareStore) DeleteAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, names ...string) error {
args := server.DeleteAttributesArgs{
Origin: origin,
ResourceID: resourceID,
Names: names,
}
reply := server.DeleteAttributesArgs{}
if err := s.call(ctx, "Service.DeleteAttributes", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// DeleteResource implements share.Store.
func (s *ShareStore) DeleteResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) error {
args := server.DeleteResourceArgs{
Origin: origin,
ResourceID: resourceID,
}
reply := server.DeleteResourceReply{}
if err := s.call(ctx, "Service.DeleteResource", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// FindResources implements share.Store.
func (s *ShareStore) FindResources(ctx context.Context, funcs ...share.FindResourcesOptionFunc) ([]share.Resource, error) {
options := share.NewFindResourcesOptions(funcs...)
args := server.FindResourcesArgs{
Options: options,
}
reply := server.FindResourcesReply{}
if err := s.call(ctx, "Service.FindResources", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
resources := make([]share.Resource, len(reply.Resources))
for idx, res := range reply.Resources {
resources[idx] = res
}
return resources, nil
}
// GetResource implements share.Store.
func (s *ShareStore) GetResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) (share.Resource, error) {
args := server.GetResourceArgs{
Origin: origin,
ResourceID: resourceID,
}
reply := server.GetResourceReply{}
if err := s.call(ctx, "Service.GetResource", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Resource, nil
}
// UpdateAttributes implements share.Store.
func (s *ShareStore) UpdateAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, attributes ...share.Attribute) (share.Resource, error) {
serializableAttributes := make([]*server.SerializableAttribute, len(attributes))
for attrIdx, attr := range attributes {
serializableAttributes[attrIdx] = server.FromAttribute(attr)
}
args := server.UpdateAttributesArgs{
Origin: origin,
ResourceID: resourceID,
Attributes: serializableAttributes,
}
reply := server.UpdateAttributesReply{}
if err := s.call(ctx, "Service.UpdateAttributes", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Resource, nil
}
func (s *ShareStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(remapShareError(err))
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *ShareStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewShareStore(url *url.URL) *ShareStore {
return &ShareStore{url}
}
var _ share.Store = &ShareStore{}

View File

@ -0,0 +1,67 @@
package client
import (
"fmt"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestShareStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
testsuite.TestStore(t, func(testName string) (share.Store, error) {
httpServer, err := startNewShareStoreServer(testName)
if err != nil {
return nil, errors.WithStack(err)
}
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
return NewShareStore(serverURL), nil
})
}
func getSQLiteShareStore(testName string) (*sqlite.ShareStore, error) {
filename := strings.ToLower(strings.ReplaceAll(testName, " ", "_"))
file := fmt.Sprintf("./testdata/sharestore_test_%s.sqlite", filename)
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewShareStore(dsn)
return store, nil
}
func startNewShareStoreServer(testName string) (*httptest.Server, error) {
store, err := getSQLiteShareStore(testName)
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewShareStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1,22 @@
package rpc
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/client"
)
func init() {
driver.RegisterDocumentStoreFactory("rpc", documentStoreFactory)
driver.RegisterBlobStoreFactory("rpc", blobStoreFactory)
}
func documentStoreFactory(url *url.URL) (storage.DocumentStore, error) {
return client.NewDocumentStore(url), nil
}
func blobStoreFactory(url *url.URL) (storage.BlobStore, error) {
return client.NewBlobStore(url), nil
}

View File

@ -0,0 +1,42 @@
package gob
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
type BlobInfo struct {
Bucket_ string
ContentType_ string
BlobID_ storage.BlobID
ModTime_ time.Time
Size_ int64
}
// Bucket implements storage.BlobInfo.
func (bi *BlobInfo) Bucket() string {
return bi.Bucket_
}
// ContentType implements storage.BlobInfo.
func (bi *BlobInfo) ContentType() string {
return bi.ContentType_
}
// ID implements storage.BlobInfo.
func (bi *BlobInfo) ID() storage.BlobID {
return bi.BlobID_
}
// ModTime implements storage.BlobInfo.
func (bi *BlobInfo) ModTime() time.Time {
return bi.ModTime_
}
// Size implements storage.BlobInfo.
func (bi *BlobInfo) Size() int64 {
return bi.Size_
}
var _ storage.BlobInfo = &BlobInfo{}

View File

@ -0,0 +1,18 @@
package gob
import (
"encoding/gob"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
func init() {
gob.Register(storage.Document{})
gob.Register(storage.DocumentID(""))
gob.Register(time.Time{})
gob.Register(map[string]interface{}{})
gob.Register([]interface{}{})
gob.Register([]map[string]interface{}{})
gob.Register(&BlobInfo{})
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseBucketArgs struct {
BucketID BucketID
}
type CloseBucketReply struct {
}
func (s *Service) CloseBucket(ctx context.Context, args *CloseBucketArgs, reply *CloseBucketReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
if err := bucket.Close(); err != nil {
return errors.WithStack(err)
}
s.buckets.Delete(args.BucketID)
*reply = CloseBucketReply{}
return nil
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseReaderArgs struct {
ReaderID ReaderID
}
type CloseReaderReply struct {
}
func (s *Service) CloseReader(ctx context.Context, args *CloseReaderArgs, reply *CloseReaderReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
if err := reader.Close(); err != nil {
return errors.WithStack(err)
}
s.readers.Delete(args.ReaderID)
*reply = CloseReaderReply{}
return nil
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseWriterArgs struct {
WriterID WriterID
}
type CloseWriterReply struct {
}
func (s *Service) CloseWriter(ctx context.Context, args *CloseWriterArgs, reply *CloseWriterReply) error {
writer, err := s.getOpenedWriter(args.WriterID)
if err != nil {
return errors.WithStack(err)
}
if err := writer.Close(); err != nil {
return errors.WithStack(err)
}
s.writers.Delete(args.WriterID)
*reply = CloseWriterReply{}
return nil
}

View File

@ -0,0 +1,22 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type DeleteBucketArgs struct {
BucketName string
}
type DeleteBucketReply struct {
}
func (s *Service) DeleteBucket(ctx context.Context, args *DeleteBucketArgs, reply *DeleteBucketReply) error {
if err := s.store.DeleteBucket(ctx, args.BucketName); err != nil {
return errors.WithStack(err)
}
return nil
}

View File

@ -0,0 +1,42 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
"github.com/pkg/errors"
)
type GetBlobInfoArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type GetBlobInfoReply struct {
BlobInfo storage.BlobInfo
}
func (s *Service) GetBlobInfo(ctx context.Context, args *GetBlobInfoArgs, reply *GetBlobInfoReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
blobInfo, err := bucket.Get(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetBlobInfoReply{
BlobInfo: &gob.BlobInfo{
Bucket_: blobInfo.Bucket(),
ContentType_: blobInfo.ContentType(),
BlobID_: blobInfo.ID(),
ModTime_: blobInfo.ModTime(),
Size_: blobInfo.Size(),
},
}
return nil
}

View File

@ -0,0 +1,33 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type GetBucketSizeArgs struct {
BucketID BucketID
}
type GetBucketSizeReply struct {
Size int64
}
func (s *Service) GetBucketSize(ctx context.Context, args *GetBucketSizeArgs, reply *GetBucketSizeReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
size, err := bucket.Size(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = GetBucketSizeReply{
Size: size,
}
return nil
}

View File

@ -0,0 +1,34 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type ListBlobInfoArgs struct {
BucketID BucketID
}
type ListBlobInfoReply struct {
BlobInfos []storage.BlobInfo
}
func (s *Service) ListBlobInfo(ctx context.Context, args *ListBlobInfoArgs, reply *ListBlobInfoReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
blobInfos, err := bucket.List(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = ListBlobInfoReply{
BlobInfos: blobInfos,
}
return nil
}

View File

@ -0,0 +1,27 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type ListBucketsArgs struct {
}
type ListBucketsReply struct {
Buckets []string
}
func (s *Service) ListBuckets(ctx context.Context, args *ListBucketsArgs, reply *ListBucketsReply) error {
buckets, err := s.store.ListBuckets(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = ListBucketsReply{
Buckets: buckets,
}
return nil
}

View File

@ -0,0 +1,57 @@
package blob
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type NewBlobReaderArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type NewBlobReaderReply struct {
ReaderID ReaderID
}
func (s *Service) NewBlobReader(ctx context.Context, args *NewBlobReaderArgs, reply *NewBlobReaderReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
readerID, err := NewReaderID()
if err != nil {
return errors.WithStack(err)
}
reader, err := bucket.NewReader(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
s.readers.Store(readerID, reader)
*reply = NewBlobReaderReply{
ReaderID: readerID,
}
return nil
}
func (s *Service) getOpenedReader(id ReaderID) (io.ReadSeekCloser, error) {
raw, exists := s.readers.Load(id)
if !exists {
return nil, errors.Errorf("could not find writer '%s'", id)
}
reader, ok := raw.(io.ReadSeekCloser)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for writer", raw)
}
return reader, nil
}

View File

@ -0,0 +1,57 @@
package blob
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type NewBlobWriterArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type NewBlobWriterReply struct {
WriterID WriterID
}
func (s *Service) NewBlobWriter(ctx context.Context, args *NewBlobWriterArgs, reply *NewBlobWriterReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
writerID, err := NewWriterID()
if err != nil {
return errors.WithStack(err)
}
writer, err := bucket.NewWriter(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
s.writers.Store(writerID, writer)
*reply = NewBlobWriterReply{
WriterID: writerID,
}
return nil
}
func (s *Service) getOpenedWriter(id WriterID) (io.WriteCloser, error) {
raw, exists := s.writers.Load(id)
if !exists {
return nil, errors.Errorf("could not find writer '%s'", id)
}
writer, ok := raw.(io.WriteCloser)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for writer", raw)
}
return writer, nil
}

View File

@ -0,0 +1,50 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type OpenBucketArgs struct {
BucketName string
}
type OpenBucketReply struct {
BucketID BucketID
}
func (s *Service) OpenBucket(ctx context.Context, args *OpenBucketArgs, reply *OpenBucketReply) error {
bucket, err := s.store.OpenBucket(ctx, args.BucketName)
if err != nil {
return errors.WithStack(err)
}
bucketID, err := NewBucketID()
if err != nil {
return errors.WithStack(err)
}
s.buckets.Store(bucketID, bucket)
*reply = OpenBucketReply{
BucketID: bucketID,
}
return nil
}
func (s *Service) getOpenedBucket(id BucketID) (storage.BlobBucket, error) {
raw, exists := s.buckets.Load(id)
if !exists {
return nil, errors.WithStack(storage.ErrBucketClosed)
}
bucket, ok := raw.(storage.BlobBucket)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for blob bucket", raw)
}
return bucket, nil
}

View File

@ -0,0 +1,41 @@
package blob
import (
"context"
"io"
"github.com/pkg/errors"
)
type ReadBlobArgs struct {
ReaderID ReaderID
Length int
}
type ReadBlobReply struct {
Data []byte
Read int
EOF bool
}
func (s *Service) ReadBlob(ctx context.Context, args *ReadBlobArgs, reply *ReadBlobReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
buff := make([]byte, args.Length)
read, err := reader.Read(buff)
if err != nil && !errors.Is(err, io.EOF) {
return errors.WithStack(err)
}
*reply = ReadBlobReply{
Read: read,
Data: buff,
EOF: errors.Is(err, io.EOF),
}
return nil
}

View File

@ -0,0 +1,38 @@
package blob
import (
"context"
"io"
"github.com/pkg/errors"
)
type SeekBlobArgs struct {
ReaderID ReaderID
Offset int64
Whence int
}
type SeekBlobReply struct {
Read int64
EOF bool
}
func (s *Service) SeekBlob(ctx context.Context, args *SeekBlobArgs, reply *SeekBlobReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
read, err := reader.Seek(args.Offset, args.Whence)
if err != nil && !errors.Is(err, io.EOF) {
return errors.WithStack(err)
}
*reply = SeekBlobReply{
Read: read,
EOF: errors.Is(err, io.EOF),
}
return nil
}

View File

@ -0,0 +1,60 @@
package blob
import (
"fmt"
"sync"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/google/uuid"
"github.com/pkg/errors"
)
type BucketID string
type WriterID string
type ReaderID string
type Service struct {
store storage.BlobStore
buckets sync.Map
writers sync.Map
readers sync.Map
}
func NewService(store storage.BlobStore) *Service {
return &Service{
store: store,
}
}
func NewBucketID() (BucketID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := BucketID(fmt.Sprintf("bucket-%s", uuid.String()))
return id, nil
}
func NewWriterID() (WriterID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := WriterID(fmt.Sprintf("writer-%s", uuid.String()))
return id, nil
}
func NewReaderID() (ReaderID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := ReaderID(fmt.Sprintf("reader-%s", uuid.String()))
return id, nil
}

View File

@ -0,0 +1,34 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type WriteBlobArgs struct {
WriterID WriterID
Data []byte
}
type WriteBlobReply struct {
Written int
}
func (s *Service) WriteBlob(ctx context.Context, args *WriteBlobArgs, reply *WriteBlobReply) error {
writer, err := s.getOpenedWriter(args.WriterID)
if err != nil {
return errors.WithStack(err)
}
written, err := writer.Write(args.Data)
if err != nil {
return errors.WithStack(err)
}
*reply = WriteBlobReply{
Written: written,
}
return nil
}

View File

@ -0,0 +1,26 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type DeleteDocumentArgs struct {
Collection string
DocumentID storage.DocumentID
}
type DeleteDocumentReply struct {
}
func (s *Service) DeleteDocument(ctx context.Context, args DeleteDocumentArgs, reply *DeleteDocumentReply) error {
if err := s.store.Delete(ctx, args.Collection, args.DocumentID); err != nil {
return errors.WithStack(err)
}
*reply = DeleteDocumentReply{}
return nil
}

View File

@ -0,0 +1,30 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type GetDocumentArgs struct {
Collection string
DocumentID storage.DocumentID
}
type GetDocumentReply struct {
Document storage.Document
}
func (s *Service) GetDocument(ctx context.Context, args GetDocumentArgs, reply *GetDocumentReply) error {
document, err := s.store.Get(ctx, args.Collection, args.DocumentID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetDocumentReply{
Document: document,
}
return nil
}

View File

@ -0,0 +1,53 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/filter"
"github.com/pkg/errors"
)
type QueryDocumentsArgs struct {
Collection string
Filter map[string]any
Options *storage.QueryOptions
}
type QueryDocumentsReply struct {
Documents []storage.Document
}
func (s *Service) QueryDocuments(ctx context.Context, args QueryDocumentsArgs, reply *QueryDocumentsReply) error {
var (
argsFilter *filter.Filter
err error
)
if args.Filter != nil {
argsFilter, err = filter.NewFrom(args.Filter)
if err != nil {
return errors.WithStack(err)
}
}
documents, err := s.store.Query(ctx, args.Collection, argsFilter, withQueryOptions(args.Options))
if err != nil {
return errors.WithStack(err)
}
*reply = QueryDocumentsReply{
Documents: documents,
}
return nil
}
func withQueryOptions(opts *storage.QueryOptions) storage.QueryOptionFunc {
return func(o *storage.QueryOptions) {
o.Limit = opts.Limit
o.Offset = opts.Offset
o.OrderBy = opts.OrderBy
o.OrderDirection = opts.OrderDirection
}
}

View File

@ -0,0 +1,11 @@
package document
import "forge.cadoles.com/arcad/edge/pkg/storage"
type Service struct {
store storage.DocumentStore
}
func NewService(store storage.DocumentStore) *Service {
return &Service{store}
}

View File

@ -0,0 +1,30 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type UpsertDocumentArgs struct {
Collection string
Document storage.Document
}
type UpsertDocumentReply struct {
Document storage.Document
}
func (s *Service) UpsertDocument(ctx context.Context, args UpsertDocumentArgs, reply *UpsertDocumentReply) error {
document, err := s.store.Upsert(ctx, args.Collection, args.Document)
if err != nil {
return errors.WithStack(err)
}
*reply = UpsertDocumentReply{
Document: document,
}
return nil
}

View File

@ -0,0 +1,5 @@
package server
import (
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
)

View File

@ -0,0 +1,29 @@
package server
import (
"github.com/keegancsmith/rpc"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/document"
shareService "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func NewBlobStoreServer(store storage.BlobStore) *rpc.Server {
server := rpc.NewServer()
server.Register(blob.NewService(store))
return server
}
func NewDocumentStoreServer(store storage.DocumentStore) *rpc.Server {
server := rpc.NewServer()
server.Register(document.NewService(store))
return server
}
func NewShareStoreServer(store share.Store) *rpc.Server {
server := rpc.NewServer()
server.Register(shareService.NewService(store))
return server
}

View File

@ -0,0 +1,28 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type DeleteAttributesArgs struct {
Origin app.ID
ResourceID share.ResourceID
Names []string
}
type DeleteAttributesReply struct {
}
func (s *Service) DeleteAttributes(ctx context.Context, args DeleteAttributesArgs, reply *DeleteAttributesReply) error {
if err := s.store.DeleteAttributes(ctx, args.Origin, args.ResourceID, args.Names...); err != nil {
return errors.WithStack(err)
}
*reply = DeleteAttributesReply{}
return nil
}

View File

@ -0,0 +1,27 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type DeleteResourceArgs struct {
Origin app.ID
ResourceID share.ResourceID
}
type DeleteResourceReply struct {
}
func (s *Service) DeleteResource(ctx context.Context, args DeleteResourceArgs, reply *DeleteResourceReply) error {
if err := s.store.DeleteResource(ctx, args.Origin, args.ResourceID); err != nil {
return errors.WithStack(err)
}
*reply = DeleteResourceReply{}
return nil
}

View File

@ -0,0 +1,41 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type FindResourcesArgs struct {
Options *share.FindResourcesOptions
}
type FindResourcesReply struct {
Resources []*SerializableResource
}
func (s *Service) FindResources(ctx context.Context, args FindResourcesArgs, reply *FindResourcesReply) error {
resources, err := s.store.FindResources(ctx, withFindResourcesOptions(args.Options))
if err != nil {
return errors.WithStack(err)
}
serializableResources := make([]*SerializableResource, len(resources))
for resIdx, r := range resources {
serializableResources[resIdx] = FromResource(r)
}
*reply = FindResourcesReply{
Resources: serializableResources,
}
return nil
}
func withFindResourcesOptions(opts *share.FindResourcesOptions) share.FindResourcesOptionFunc {
return func(o *share.FindResourcesOptions) {
o.Name = opts.Name
o.ValueType = opts.ValueType
}
}

View File

@ -0,0 +1,31 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type GetResourceArgs struct {
Origin app.ID
ResourceID share.ResourceID
}
type GetResourceReply struct {
Resource *SerializableResource
}
func (s *Service) GetResource(ctx context.Context, args GetResourceArgs, reply *GetResourceReply) error {
resource, err := s.store.GetResource(ctx, args.Origin, args.ResourceID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetResourceReply{
Resource: FromResource(resource),
}
return nil
}

View File

@ -0,0 +1,94 @@
package share
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func FromResource(res share.Resource) *SerializableResource {
serializableAttributes := make([]*SerializableAttribute, len(res.Attributes()))
for attrIdx, attr := range res.Attributes() {
serializableAttributes[attrIdx] = FromAttribute(attr)
}
return &SerializableResource{
ID_: res.ID(),
Origin_: res.Origin(),
Attributes_: serializableAttributes,
}
}
func FromAttribute(attr share.Attribute) *SerializableAttribute {
return &SerializableAttribute{
Name_: attr.Name(),
Value_: attr.Value(),
Type_: attr.Type(),
UpdatedAt_: attr.UpdatedAt(),
CreatedAt_: attr.CreatedAt(),
}
}
type SerializableResource struct {
ID_ share.ResourceID
Origin_ app.ID
Attributes_ []*SerializableAttribute
}
// Attributes implements share.Resource.
func (r *SerializableResource) Attributes() []share.Attribute {
attributes := make([]share.Attribute, len(r.Attributes_))
for idx, attr := range r.Attributes_ {
attributes[idx] = attr
}
return attributes
}
// ID implements share.Resource.
func (r *SerializableResource) ID() share.ResourceID {
return r.ID_
}
// Origin implements share.Resource.
func (r *SerializableResource) Origin() app.ID {
return r.Origin_
}
var _ share.Resource = &SerializableResource{}
type SerializableAttribute struct {
Name_ string
Value_ any
Type_ share.ValueType
UpdatedAt_ time.Time
CreatedAt_ time.Time
}
// CreatedAt implements share.Attribute.
func (a *SerializableAttribute) CreatedAt() time.Time {
return a.CreatedAt_
}
// Name implements share.Attribute.
func (a *SerializableAttribute) Name() string {
return a.Name_
}
// Type implements share.Attribute.
func (a *SerializableAttribute) Type() share.ValueType {
return a.Type_
}
// UpdatedAt implements share.Attribute.
func (a *SerializableAttribute) UpdatedAt() time.Time {
return a.UpdatedAt_
}
// Value implements share.Attribute.
func (a *SerializableAttribute) Value() any {
return a.Value_
}
var _ share.Attribute = &SerializableAttribute{}

View File

@ -0,0 +1,13 @@
package share
import (
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
type Service struct {
store share.Store
}
func NewService(store share.Store) *Service {
return &Service{store}
}

View File

@ -0,0 +1,37 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type UpdateAttributesArgs struct {
Origin app.ID
ResourceID share.ResourceID
Attributes []*SerializableAttribute
}
type UpdateAttributesReply struct {
Resource *SerializableResource
}
func (s *Service) UpdateAttributes(ctx context.Context, args UpdateAttributesArgs, reply *UpdateAttributesReply) error {
attributes := make([]share.Attribute, len(args.Attributes))
for idx, attr := range args.Attributes {
attributes[idx] = attr
}
resource, err := s.store.UpdateAttributes(ctx, args.Origin, args.ResourceID, attributes...)
if err != nil {
return errors.WithStack(err)
}
*reply = UpdateAttributesReply{
Resource: FromResource(resource),
}
return nil
}

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
var shareStoreFactories = make(map[string]ShareStoreFactory, 0)
type ShareStoreFactory func(url *url.URL) (share.Store, error)
func RegisterShareStoreFactory(scheme string, factory ShareStoreFactory) {
shareStoreFactories[scheme] = factory
}
func NewShareStore(dsn string) (share.Store, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := shareStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -16,7 +16,7 @@ import (
type BlobBucket struct { type BlobBucket struct {
name string name string
getDB getDBFunc getDB GetDBFunc
closed bool closed bool
} }
@ -236,7 +236,7 @@ func (b *BlobBucket) withTx(ctx context.Context, fn func(tx *sql.Tx) error) erro
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := withTx(ctx, db, fn); err != nil { if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -246,7 +246,7 @@ func (b *BlobBucket) withTx(ctx context.Context, fn func(tx *sql.Tx) error) erro
type blobWriterCloser struct { type blobWriterCloser struct {
id storage.BlobID id storage.BlobID
bucket string bucket string
getDB getDBFunc getDB GetDBFunc
buf bytes.Buffer buf bytes.Buffer
closed bool closed bool
} }
@ -335,7 +335,7 @@ func (wbc *blobWriterCloser) withTx(ctx context.Context, fn func(tx *sql.Tx) err
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := withTx(ctx, db, fn); err != nil { if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -345,7 +345,7 @@ func (wbc *blobWriterCloser) withTx(ctx context.Context, fn func(tx *sql.Tx) err
type blobReaderCloser struct { type blobReaderCloser struct {
id storage.BlobID id storage.BlobID
bucket string bucket string
getDB getDBFunc getDB GetDBFunc
reader bytes.Reader reader bytes.Reader
once sync.Once once sync.Once
closed bool closed bool
@ -444,7 +444,7 @@ func (brc *blobReaderCloser) withTx(ctx context.Context, fn func(tx *sql.Tx) err
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := withTx(ctx, db, fn); err != nil { if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }

View File

@ -10,7 +10,7 @@ import (
) )
type BlobStore struct { type BlobStore struct {
getDB getDBFunc getDB GetDBFunc
} }
// DeleteBucket implements storage.BlobStore // DeleteBucket implements storage.BlobStore
@ -81,7 +81,7 @@ func (s *BlobStore) OpenBucket(ctx context.Context, name string) (storage.BlobBu
func ensureBlobTables(ctx context.Context, db *sql.DB) error { func ensureBlobTables(ctx context.Context, db *sql.DB) error {
logger.Debug(ctx, "creating blobs table") logger.Debug(ctx, "creating blobs table")
err := withTx(ctx, db, func(tx *sql.Tx) error { err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := ` query := `
CREATE TABLE IF NOT EXISTS blobs ( CREATE TABLE IF NOT EXISTS blobs (
id TEXT, id TEXT,
@ -114,7 +114,7 @@ func (s *BlobStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := withTx(ctx, db, fn); err != nil { if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
@ -122,13 +122,13 @@ func (s *BlobStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error
} }
func NewBlobStore(dsn string) *BlobStore { func NewBlobStore(dsn string) *BlobStore {
getDB := newGetDBFunc(dsn, ensureBlobTables) getDB := NewGetDBFunc(dsn, ensureBlobTables)
return &BlobStore{getDB} return &BlobStore{getDB}
} }
func NewBlobStoreWithDB(db *sql.DB) *BlobStore { func NewBlobStoreWithDB(db *sql.DB) *BlobStore {
getDB := newGetDBFuncFromDB(db, ensureBlobTables) getDB := NewGetDBFuncFromDB(db, ensureBlobTables)
return &BlobStore{getDB} return &BlobStore{getDB}
} }

View File

@ -0,0 +1,46 @@
package sqlite
import (
"context"
"fmt"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestBlobStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%+v", errors.WithStack(err))
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewBlobStore(dsn)
testsuite.TestBlobStore(context.Background(), t, store)
}
func BenchmarkBlobStore(t *testing.B) {
logger.SetLevel(logger.LevelError)
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%+v", errors.WithStack(err))
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewBlobStore(dsn)
testsuite.BenchmarkBlobStore(t, store)
}

View File

@ -17,7 +17,7 @@ import (
) )
type DocumentStore struct { type DocumentStore struct {
getDB getDBFunc getDB GetDBFunc
} }
// Delete implements storage.DocumentStore // Delete implements storage.DocumentStore
@ -269,15 +269,15 @@ func (s *DocumentStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) e
return errors.WithStack(err) return errors.WithStack(err)
} }
if err := withTx(ctx, db, fn); err != nil { if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err) return errors.WithStack(err)
} }
return nil return nil
} }
func ensureTables(ctx context.Context, db *sql.DB) error { func ensureDocumentTables(ctx context.Context, db *sql.DB) error {
err := withTx(ctx, db, func(tx *sql.Tx) error { err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := ` query := `
CREATE TABLE IF NOT EXISTS documents ( CREATE TABLE IF NOT EXISTS documents (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
@ -344,7 +344,7 @@ func withLimitOffsetClause(query string, args []any, limit int, offset int) (str
} }
func NewDocumentStore(path string) *DocumentStore { func NewDocumentStore(path string) *DocumentStore {
getDB := newGetDBFunc(path, ensureTables) getDB := NewGetDBFunc(path, ensureDocumentTables)
return &DocumentStore{ return &DocumentStore{
getDB: getDB, getDB: getDB,
@ -352,7 +352,7 @@ func NewDocumentStore(path string) *DocumentStore {
} }
func NewDocumentStoreWithDB(db *sql.DB) *DocumentStore { func NewDocumentStoreWithDB(db *sql.DB) *DocumentStore {
getDB := newGetDBFuncFromDB(db, ensureTables) getDB := NewGetDBFuncFromDB(db, ensureDocumentTables)
return &DocumentStore{ return &DocumentStore{
getDB: getDB, getDB: getDB,

View File

@ -1,6 +1,7 @@
package sqlite package sqlite
import ( import (
"context"
"fmt" "fmt"
"os" "os"
"testing" "testing"
@ -24,5 +25,5 @@ func TestDocumentStore(t *testing.T) {
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds()) dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewDocumentStore(dsn) store := NewDocumentStore(dsn)
testsuite.TestDocumentStore(t, store) testsuite.TestDocumentStore(context.Background(), t, store)
} }

View File

@ -0,0 +1,75 @@
package sqlite
import (
"net/url"
"os"
"path/filepath"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
func init() {
driver.RegisterDocumentStoreFactory("sqlite", documentStoreFactory)
driver.RegisterBlobStoreFactory("sqlite", blobStoreFactory)
driver.RegisterShareStoreFactory("sqlite", shareStoreFactory)
}
func documentStoreFactory(url *url.URL) (storage.DocumentStore, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewDocumentStoreWithDB(db), nil
}
func blobStoreFactory(url *url.URL) (storage.BlobStore, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewBlobStoreWithDB(db), nil
}
func shareStoreFactory(url *url.URL) (share.Store, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewShareStoreWithDB(db), nil
}

View File

@ -0,0 +1,428 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type ShareStore struct {
getDB GetDBFunc
}
// DeleteAttributes implements share.Repository
func (s *ShareStore) DeleteAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, names ...string) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
DELETE FROM resources
WHERE origin = $1 AND resource_id = $2
`
args := []any{origin, resourceID}
criteria := ""
for idx, name := range names {
if idx == 0 {
criteria += " AND ("
}
if idx != 0 {
criteria += " OR "
}
criteria += fmt.Sprintf(" name = $%d", len(args)+1)
args = append(args, name)
if idx == len(names)-1 {
criteria += " )"
}
}
query += criteria
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
res, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
affected, err := res.RowsAffected()
if err != nil {
return errors.WithStack(err)
}
if affected == 0 {
return errors.WithStack(share.ErrNotFound)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return err
}
// DeleteResource implements share.Repository
func (s *ShareStore) DeleteResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
DELETE FROM resources
WHERE origin = $1 AND resource_id = $2
`
args := []any{origin, resourceID}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
res, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
affected, err := res.RowsAffected()
if err != nil {
return errors.WithStack(err)
}
if affected == 0 {
return errors.WithStack(share.ErrNotFound)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return err
}
// FindResources implements share.Repository
func (s *ShareStore) FindResources(ctx context.Context, funcs ...share.FindResourcesOptionFunc) ([]share.Resource, error) {
opts := share.NewFindResourcesOptions(funcs...)
var resources []share.Resource
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
SELECT
main.origin, main.resource_id,
main.name, main.type, main.value,
main.created_at, main.updated_at
FROM resources AS main
JOIN resources AS sub ON
main.resource_id = sub.resource_id
AND main.origin = sub.origin
`
criteria := " WHERE 1 = 1"
preparedArgIndex := 1
args := make([]any, 0)
if opts.Name != nil {
criteria += fmt.Sprintf(" AND sub.name = $%d", preparedArgIndex)
args = append(args, *opts.Name)
preparedArgIndex++
}
if opts.ValueType != nil {
criteria += fmt.Sprintf(" AND sub.type = $%d", preparedArgIndex)
args = append(args, *opts.ValueType)
preparedArgIndex++
}
query += criteria
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
indexedResources := make(map[string]*share.BaseResource)
for rows.Next() {
var (
origin string
resourceID string
name string
valueType string
value any
updatedAt time.Time
createdAt time.Time
)
if err := rows.Scan(&origin, &resourceID, &name, &valueType, &value, &createdAt, &updatedAt); err != nil {
return errors.WithStack(err)
}
resourceKey := origin + resourceID
resource, exists := indexedResources[resourceKey]
if !exists {
resource = share.NewBaseResource(app.ID(origin), share.ResourceID(resourceID))
indexedResources[resourceKey] = resource
}
attr := share.NewBaseAttribute(
name,
share.ValueType(valueType),
value,
)
attr.SetCreatedAt(createdAt)
attr.SetUpdatedAt(updatedAt)
resource.SetAttribute(attr)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
resources = make([]share.Resource, 0, len(indexedResources))
for _, res := range indexedResources {
resources = append(resources, res)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resources, nil
}
// GetResource implements share.Repository
func (s *ShareStore) GetResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) (share.Resource, error) {
var (
resource *share.BaseResource
err error
)
err = s.withTx(ctx, func(tx *sql.Tx) error {
resource, err = s.getResourceWithinTx(ctx, tx, origin, resourceID)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resource, nil
}
// UpdateAttributes implements share.Repository
func (s *ShareStore) UpdateAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, attributes ...share.Attribute) (share.Resource, error) {
if len(attributes) == 0 {
return nil, errors.WithStack(share.ErrAttributeRequired)
}
var resource *share.BaseResource
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
INSERT INTO resources (origin, resource_id, name, type, value, created_at, updated_at)
VALUES($1, $2, $3, $4, $5, $6, $6)
ON CONFLICT (origin, resource_id, name) DO UPDATE SET
type = $4, value = $5, updated_at = $6
`
stmt, err := tx.PrepareContext(ctx, query)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := stmt.Close(); err != nil {
logger.Error(ctx, "could not close statement", logger.E(errors.WithStack(err)))
}
}()
now := time.Now().UTC()
for _, attr := range attributes {
args := []any{
string(origin), string(resourceID),
attr.Name(), string(attr.Type()), attr.Value(),
now, now,
}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
if _, err := stmt.ExecContext(ctx, args...); err != nil {
return errors.WithStack(err)
}
}
resource, err = s.getResourceWithinTx(ctx, tx, origin, resourceID)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resource, nil
}
func (s *ShareStore) getResourceWithinTx(ctx context.Context, tx *sql.Tx, origin app.ID, resourceID share.ResourceID) (*share.BaseResource, error) {
query := `
SELECT name, type, value, created_at, updated_at
FROM resources
WHERE origin = $1 AND resource_id = $2
`
rows, err := tx.QueryContext(ctx, query, origin, resourceID)
if err != nil {
return nil, errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
attributes := make([]share.Attribute, 0)
for rows.Next() {
var (
name string
valueType string
value any
updatedAt time.Time
createdAt time.Time
)
if err := rows.Scan(&name, &valueType, &value, &createdAt, &updatedAt); err != nil {
return nil, errors.WithStack(err)
}
attr := share.NewBaseAttribute(
name,
share.ValueType(valueType),
value,
)
attr.SetCreatedAt(createdAt)
attr.SetUpdatedAt(updatedAt)
attributes = append(attributes, attr)
}
if err := rows.Err(); err != nil {
return nil, errors.WithStack(err)
}
if len(attributes) == 0 {
return nil, errors.WithStack(share.ErrNotFound)
}
resource := share.NewBaseResource(origin, resourceID, attributes...)
return resource, nil
}
func (s *ShareStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
var db *sql.DB
db, err := s.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
func ensureShareTables(ctx context.Context, db *sql.DB) error {
err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := `
CREATE TABLE IF NOT EXISTS resources (
resource_id TEXT NOT NULL,
origin TEXT NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL,
value TEXT,
created_at TIMESTAMP NOT NULL,
updated_at TIMESTAMP NOT NULL,
UNIQUE(origin, resource_id, name) ON CONFLICT REPLACE
);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
query = `
CREATE INDEX IF NOT EXISTS resource_idx ON resources (origin, resource_id, name);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func NewShareStore(path string) *ShareStore {
getDB := NewGetDBFunc(path, ensureShareTables)
return &ShareStore{
getDB: getDB,
}
}
func NewShareStoreWithDB(db *sql.DB) *ShareStore {
getDB := NewGetDBFuncFromDB(db, ensureShareTables)
return &ShareStore{
getDB: getDB,
}
}
var _ share.Store = &ShareStore{}

Some files were not shown because too many files have changed in this diff Show More