Compare commits

..

27 Commits

Author SHA1 Message Date
f4a7366aad feat(storage): rpc driver client pooling and memory-constrained cache
All checks were successful
arcad/edge/pipeline/head This commit looks good
driver

ref #20
2023-11-29 11:10:29 +01:00
02c74b6f8d feat(client): add loader for apps menu
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-25 21:27:41 +02:00
8889694125 feat(cli): add basic bundle info command
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-24 22:52:51 +02:00
6a99409a15 feat(blobstore): add cache driver 2023-10-24 22:52:33 +02:00
2fc590d708 feat(storage): retry sqlite failed transaction when database is busy
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-22 23:18:02 +02:00
6e4bf2f025 feat(storage): remap rpc errors
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-22 23:04:56 +02:00
22a3326be9 feat(lifecycle): execute onInit func asynchronously
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-22 10:47:44 +02:00
0cfb132b65 feat(lifecycle-module): add debug message for onInit() execution
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-21 21:46:51 +02:00
de4ab0d02c fix(bus): prevent double close in event dispatcher
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-21 21:38:34 +02:00
d1458bab4a ci: use go 1.21.2
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-20 11:01:32 +02:00
a5c67c29d0 feat(bundle): add zim format support
Some checks failed
arcad/edge/pipeline/head There was a failure building this commit
2023-10-19 22:20:52 +02:00
1544212ab5 feat: capture logged exceptions and integrate storage-server with sentry 2023-10-19 21:47:09 +02:00
efb8ba8b99 feat(app): pass context to start process 2023-10-19 20:05:59 +02:00
4d064de164 feat(filter): add basic json parsing test case
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-11 11:43:17 +02:00
8a5a1cd482 chore: watch .env file in dev mode
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-03 11:25:14 -06:00
3fd25988cf feat(storage-server): fix typo in openrc init script 2023-10-03 11:24:59 -06:00
ebe3e77879 feat(storage-server): remove /var/log/storage-server directory for apk packager 2023-10-03 11:24:35 -06:00
3078ea7d21 feat(storage-server): add check-token command 2023-10-03 11:24:03 -06:00
4c6e979bb6 fix(ci): inject current branch name in release tasks
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-02 21:25:36 -06:00
0fded0170a feat(storage-server): fix service
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-02 20:56:53 -06:00
6ddd831025 fix(ci): update sdk test app version correctly
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-10-02 16:11:46 -06:00
4fe68e335a fix(ci): add missing task dependency
Some checks failed
arcad/edge/pipeline/head There was a failure building this commit
2023-10-02 15:18:23 -06:00
599ff749d3 Merge pull request 'feat(storage): rpc based implementation' (#8) from rpc-store into master
Some checks failed
arcad/edge/pipeline/head There was a failure building this commit
Reviewed-on: #8
2023-10-02 23:14:21 +02:00
9f89c89fb9 feat(storage-server): add packaging services
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-10-02 15:05:18 -06:00
d2472623f2 feat(storage-server): jwt based authentication
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-10-01 19:56:38 -06:00
c63af872ea feat: goreleaser packaging
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-09-28 14:26:46 -06:00
8e574c299b feat(storage): rpc based implementation
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good
2023-09-28 12:36:30 -06:00
206 changed files with 7674 additions and 758 deletions

View File

@ -1 +1,4 @@
RUN_APP_ARGS=""
RUN_APP_ARGS=""
#EDGE_DOCUMENTSTORE_DSN="rpc://localhost:3001/documentstore?tenant=local&appId=%APPID%"
#EDGE_BLOBSTORE_DSN="cache://localhost:3001/blobstore?driver=rpc&tenant=local&appId=%APPID%"
#EDGE_SHARESTORE_DSN="rpc://localhost:3001/sharestore?tenant=local"

8
.gitignore vendored
View File

@ -4,4 +4,10 @@
/tools
*.sqlite
/.gitea-release
/.edge
/.edge
/data
.mktools/
/dist
/.chglog
/CHANGELOG.md
/storage-server.key

117
.goreleaser.yml Normal file
View File

@ -0,0 +1,117 @@
project_name: edge
before:
hooks:
- go mod tidy
- go generate ./...
builds:
- id: edge-cli
binary: edge-cli
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
main: ./cmd/cli
- id: storage-server
binary: storage-server
env:
- CGO_ENABLED=0
ldflags:
- -s
- -w
gcflags:
- -trimpath="${PWD}"
asmflags:
- -trimpath="${PWD}"
goos:
- linux
goarch:
- amd64
main: ./cmd/storage-server
archives:
- id: edge-cli
builds: ["edge-cli"]
name_template: 'edge-cli_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
- id: storage-server
builds: ["storage-server"]
name_template: 'storage-server_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}{{ with .Mips }}_{{ . }}{{ end }}{{ if not (eq .Amd64 "v1") }}{{ .Amd64 }}{{ end }}'
files:
- README.md
- CHANGELOG.md
checksum:
name_template: 'checksums.txt'
snapshot:
name_template: "{{ .Version }}"
changelog:
sort: asc
filters:
exclude:
- '^docs:'
- '^test:'
nfpms:
- id: edge-cli
builds:
- "edge-cli"
package_name: edge-cli
homepage: https://forge.cadoles.com/arcad/edge
maintainer: William Petit <wpetit@cadoles.com>
description: |-
license: AGPL-3.0
formats:
- apk
- deb
- id: storage-server
builds:
- "storage-server"
package_name: storage-server
homepage: https://forge.cadoles.com/arcad/edge
maintainer: William Petit <wpetit@cadoles.com>
description: |-
license: AGPL-3.0
formats:
- apk
- deb
contents:
# Deb
- src: misc/packaging/systemd/storage-server.systemd.service
dst: /usr/lib/systemd/system/storage-server.service
packager: deb
- src: misc/packaging/systemd/storage-server.env
dst: /etc/storage-server/environ
type: config|noreplace
file_info:
mode: 0640
packager: deb
# APK
- src: misc/packaging/openrc/storage-server.openrc.sh
dst: /etc/init.d/storage-server
file_info:
mode: 0755
packager: apk
- src: misc/packaging/openrc/storage-server.conf
type: config|noreplace
dst: /etc/conf.d/storage-server
file_info:
mode: 0640
packager: apk
- dst: /var/lib/storage-server
type: dir
file_info:
mode: 0700
packager: apk
scripts:
postinstall: "misc/packaging/common/postinstall-storage-server.sh"

3
Jenkinsfile vendored
View File

@ -34,7 +34,8 @@ pipeline {
passwordVariable: 'GITEA_RELEASE_PASSWORD'
])
]) {
sh 'make gitea-release'
sh 'make .mktools'
sh "export MKT_PROJECT_VERSION_BRANCH_NAME=${env.BRANCH_NAME}; make gitea-release"
}
}
}

View File

@ -6,14 +6,17 @@ GOTEST_ARGS ?= -short -timeout 60s
ESBUILD_VERSION ?= v0.17.5
GIT_VERSION := $(shell git describe --always)
DATE_VERSION := $(shell date +%Y.%-m.%-d)
FULL_VERSION := v$(DATE_VERSION)-$(GIT_VERSION)$(if $(shell git diff --stat),-dirty,)
APP_PATH ?= misc/client-sdk-testsuite/dist
RUN_APP_ARGS ?=
RUN_STORAGE_SERVER_ARGS ?=
GORELEASER_VERSION ?= v1.21.2
GORELEASER_ARGS ?= release --snapshot --clean
SHELL := bash
build: build-edge-cli build-client-sdk-test-app
build: build-cli build-storage-server build-client-sdk-test-app
watch: tools/modd/bin/modd
tools/modd/bin/modd
@ -22,17 +25,23 @@ watch: tools/modd/bin/modd
test: test-go
test-go:
go test -v -count=1 $(GOTEST_ARGS) ./...
go test -count=1 $(GOTEST_ARGS) ./...
lint:
golangci-lint run --enable-all $(LINT_ARGS)
build-edge-cli: build-sdk
build-cli: build-sdk
CGO_ENABLED=0 go build \
-v \
-o ./bin/cli \
./cmd/cli
build-storage-server: build-sdk
CGO_ENABLED=0 go build \
-v \
-o ./bin/storage-server \
./cmd/storage-server
build-client-sdk-test-app:
cd misc/client-sdk-testsuite && $(MAKE) dist
@ -68,25 +77,31 @@ node_modules:
run-app: .env
( set -o allexport && source .env && set +o allexport && bin/cli app run -p $(APP_PATH) $$RUN_APP_ARGS )
run-storage-server: .env
( set -o allexport && source .env && set +o allexport && bin/storage-server run $$RUN_STORAGE_SERVER_ARGS )
.env:
cp .env.dist .env
gitea-release: tools/yq/bin/yq tools/gitea-release/bin/gitea-release.sh build
gitea-release: .mktools tools/yq/bin/yq tools/gitea-release/bin/gitea-release.sh goreleaser build
mkdir -p .gitea-release
rm -rf .gitea-release/*
cp bin/cli .gitea-release/edge_cli_amd64
cp dist/*.deb .gitea-release/
cp dist/*.tar.gz .gitea-release/
cp dist/*.apk .gitea-release/
cp CHANGELOG.md .gitea-release/
# Create client-sdk-testsuite package
tools/yq/bin/yq -i '.version = "$(FULL_VERSION)"' ./misc/client-sdk-testsuite/dist/manifest.yml
.gitea-release/edge_cli_amd64 app package -d ./misc/client-sdk-testsuite/dist -o .gitea-release
tools/yq/bin/yq -i '.version = "$(MKT_PROJECT_VERSION)"' ./misc/client-sdk-testsuite/dist/manifest.yml
bin/cli app package -d ./misc/client-sdk-testsuite/dist -o .gitea-release
GITEA_RELEASE_PROJECT="edge" \
GITEA_RELEASE_ORG="arcad" \
GITEA_RELEASE_BASE_URL="https://forge.cadoles.com" \
GITEA_RELEASE_VERSION="$(FULL_VERSION)" \
GITEA_RELEASE_NAME="$(FULL_VERSION)" \
GITEA_RELEASE_COMMITISH_TARGET="$(GIT_VERSION)" \
GITEA_RELEASE_VERSION="$(MKT_PROJECT_VERSION)" \
GITEA_RELEASE_NAME="$(MKT_PROJECT_VERSION)" \
GITEA_RELEASE_COMMITISH_TARGET="$$(git rev-parse HEAD)" \
GITEA_RELEASE_IS_DRAFT="false" \
GITEA_RELEASE_IS_PRERELEASE="true" \
GITEA_RELEASE_BODY="" \
@ -105,4 +120,22 @@ tools/yq/bin/yq:
tools/modd/bin/modd:
mkdir -p tools/modd/bin
GOBIN=$(PWD)/tools/modd/bin go install -mod=readonly github.com/cortesi/modd/cmd/modd@latest
GOBIN=$(PWD)/tools/modd/bin go install -mod=readonly github.com/cortesi/modd/cmd/modd@latest
.PHONY: goreleaser
goreleaser: .env .mktools changelog
( set -o allexport && source .env && set +o allexport && curl -sfL https://goreleaser.com/static/run | VERSION="$(GORELEASER_VERSION)" GORELEASER_CURRENT_TAG="$$MKT_PROJECT_VERSION" bash /dev/stdin $(GORELEASER_ARGS) )
.PHONY: changelog
changelog: .mktools
$(MAKE) MKT_GIT_CHGLOG_ARGS='--next-tag "$(MKT_PROJECT_VERSION)" --tag-filter-pattern "$(MKT_PROJECT_VERSION_CHANNEL)" --output CHANGELOG.md' mkt-changelog
.PHONY: mktools
mktools:
rm -rf .mktools
curl -q https://forge.cadoles.com/Cadoles/mktools/raw/branch/master/install.sh | $(SHELL)
.mktools:
$(MAKE) mktools
-include .mktools/*.mk

View File

@ -0,0 +1,56 @@
package app
import (
"os"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bundle"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v2"
)
func InfoCommand() *cli.Command {
return &cli.Command{
Name: "info",
Usage: "Print app manifest informations",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "path",
Usage: "use `PATH` as app bundle (zip, zim or directory bundle)",
Aliases: []string{"p"},
Value: "",
Required: true,
},
},
Action: func(ctx *cli.Context) error {
appPath := ctx.String("path")
bundle, err := bundle.FromPath(appPath)
if err != nil {
return errors.Wrap(err, "could not load app bundle")
}
manifest, err := app.LoadManifest(bundle)
if err != nil {
return errors.Wrap(err, "could not load app manifest")
}
if valid, err := manifest.Validate(manifestMetadataValidators...); !valid {
return errors.Wrap(err, "invalid app manifest")
}
encoder := yaml.NewEncoder(os.Stdout)
if err := encoder.Encode(manifest); err != nil {
return errors.Wrap(err, "could not encode manifest")
}
if err := encoder.Close(); err != nil {
return errors.WithStack(err)
}
return nil
},
}
}

View File

@ -12,6 +12,7 @@ func Root() *cli.Command {
RunCommand(),
PackageCommand(),
HashPasswordCommand(),
InfoCommand(),
},
}
}

View File

@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
@ -17,6 +16,7 @@ import (
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
appHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/module"
appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
appModuleMemory "forge.cadoles.com/arcad/edge/pkg/module/app/memory"
@ -28,9 +28,7 @@ import (
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
shareSqlite "forge.cadoles.com/arcad/edge/pkg/module/share/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage"
storageSqlite "forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/bundle"
@ -45,8 +43,19 @@ import (
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/argon2id"
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/plain"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
// Register storage drivers
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
var dummySecret = []byte("not_so_secret")
func RunCommand() *cli.Command {
return &cli.Command{
Name: "run",
@ -75,14 +84,22 @@ func RunCommand() *cli.Command {
Value: 0,
},
&cli.StringFlag{
Name: "storage-file",
Usage: "use `FILE` for SQLite storage database",
Value: ".edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
Name: "blobstore-dsn",
Usage: "use `DSN` for blob storage",
EnvVars: []string{"EDGE_BLOBSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "shared-resources-file",
Usage: "use `FILE` for SQLite shared resources database",
Value: ".edge/shared-resources.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
Name: "documentstore-dsn",
Usage: "use `DSN` for document storage",
EnvVars: []string{"EDGE_DOCUMENTSTORE_DSN"},
Value: "sqlite://.edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "sharestore-dsn",
Usage: "use `DSN` for share storage",
EnvVars: []string{"EDGE_SHARESTORE_DSN"},
Value: "sqlite://.edge/share.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "accounts-file",
@ -96,9 +113,10 @@ func RunCommand() *cli.Command {
logFormat := ctx.String("log-format")
logLevel := ctx.Int("log-level")
storageFile := ctx.String("storage-file")
blobstoreDSN := ctx.String("blobstore-dsn")
documentstoreDSN := ctx.String("documentstore-dsn")
shareStoreDSN := ctx.String("sharestore-dsn")
accountsFile := ctx.String("accounts-file")
sharedResourcesFile := ctx.String("shared-resources-file")
logger.SetFormat(logger.Format(logFormat))
logger.SetLevel(logger.Level(logLevel))
@ -144,8 +162,8 @@ func RunCommand() *cli.Command {
appCtx := logger.With(cmdCtx, logger.F("address", address))
if err := runApp(appCtx, path, address, storageFile, accountsFile, appsRepository, sharedResourcesFile); err != nil {
logger.Error(appCtx, "could not run app", logger.E(errors.WithStack(err)))
if err := runApp(appCtx, path, address, documentstoreDSN, blobstoreDSN, shareStoreDSN, accountsFile, appsRepository); err != nil {
logger.Error(appCtx, "could not run app", logger.CapturedE(errors.WithStack(err)))
}
}(p, port, idx)
}
@ -157,7 +175,7 @@ func RunCommand() *cli.Command {
}
}
func runApp(ctx context.Context, path string, address string, storageFile string, accountsFile string, appRepository appModule.Repository, sharedResourcesFile string) error {
func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN, shareStoreDSN, accountsFile string, appRepository appModule.Repository) error {
absPath, err := filepath.Abs(path)
if err != nil {
return errors.Wrapf(err, "could not resolve path '%s'", path)
@ -182,17 +200,17 @@ func runApp(ctx context.Context, path string, address string, storageFile string
ctx = logger.With(ctx, logger.F("appID", manifest.ID))
// Add auth handler
key, err := dummyKey()
key, err := jwtutil.NewSymmetricKey(dummySecret)
if err != nil {
return errors.WithStack(err)
}
deps := &moduleDeps{}
funcs := []ModuleDepFunc{
initAppID(manifest),
initMemoryBus,
initDatastores(storageFile, manifest.ID),
initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN, manifest.ID),
initAccounts(accountsFile, manifest.ID),
initShareRepository(sharedResourcesFile),
initAppRepository(appRepository),
}
@ -209,20 +227,21 @@ func runApp(ctx context.Context, path string, address string, storageFile string
appModule.Mount(appRepository),
authModule.Mount(
authHTTP.NewLocalHandler(
jwa.HS256, key,
key,
jwa.HS256,
authHTTP.WithRoutePrefix("/auth"),
authHTTP.WithAccounts(deps.Accounts...),
),
authModule.WithJWT(dummyKeySet),
authModule.WithJWT(func() (jwk.Set, error) {
return jwtutil.NewSymmetricKeySet(dummySecret)
}),
),
),
appHTTP.WithHTTPMiddlewares(
authModuleMiddleware.AnonymousUser(
jwa.HS256, key,
),
authModuleMiddleware.AnonymousUser(key, jwa.HS256),
),
)
if err := handler.Load(bundle); err != nil {
if err := handler.Load(ctx, bundle); err != nil {
return errors.Wrap(err, "could not load app bundle")
}
@ -243,13 +262,13 @@ func runApp(ctx context.Context, path string, address string, storageFile string
}
type moduleDeps struct {
AppID app.ID
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
AppRepository appModule.Repository
ShareRepository shareModule.Repository
Accounts []authHTTP.LocalAccount
AppID app.ID
Bus bus.Bus
DocumentStore storage.DocumentStore
BlobStore storage.BlobStore
AppRepository appModule.Repository
ShareStore share.Store
Accounts []authHTTP.LocalAccount
}
type ModuleDepFunc func(*moduleDeps) error
@ -265,44 +284,16 @@ func getServerModules(deps *moduleDeps) []app.ServerModuleFactory {
module.StoreModuleFactory(deps.DocumentStore),
blob.ModuleFactory(deps.Bus, deps.BlobStore),
authModule.ModuleFactory(
authModule.WithJWT(dummyKeySet),
authModule.WithJWT(func() (jwk.Set, error) {
return jwtutil.NewSymmetricKeySet(dummySecret)
}),
),
appModule.ModuleFactory(deps.AppRepository),
fetch.ModuleFactory(deps.Bus),
shareModule.ModuleFactory(deps.AppID, deps.ShareRepository),
shareModule.ModuleFactory(deps.AppID, deps.ShareStore),
}
}
var dummySecret = []byte("not_so_secret")
func dummyKey() (jwk.Key, error) {
key, err := jwk.FromRaw(dummySecret)
if err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}
func dummyKeySet() (jwk.Set, error) {
key, err := dummyKey()
if err != nil {
return nil, errors.WithStack(err)
}
if err := key.Set(jwk.AlgorithmKey, jwa.HS256); err != nil {
return nil, errors.WithStack(err)
}
set := jwk.NewSet()
if err := set.AddKey(key); err != nil {
return nil, errors.WithStack(err)
}
return set, nil
}
func ensureDir(path string) error {
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
return errors.WithStack(err)
@ -323,10 +314,10 @@ func loadLocalAccounts(path string) ([]authHTTP.LocalAccount, error) {
return nil, errors.WithStack(err)
}
data, err := ioutil.ReadFile(path)
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
if err := ioutil.WriteFile(path, defaultAccounts, 0o640); err != nil {
if err := os.WriteFile(path, defaultAccounts, 0o640); err != nil {
return nil, errors.WithStack(err)
}
@ -366,7 +357,7 @@ func findMatchingDeviceAddress(ctx context.Context, from string, defaultAddr str
if err != nil {
logger.Error(
ctx, "could not retrieve iface adresses",
logger.E(errors.WithStack(err)), logger.F("iface", ifa.Name),
logger.CapturedE(errors.WithStack(err)), logger.F("iface", ifa.Name),
)
continue
@ -377,7 +368,7 @@ func findMatchingDeviceAddress(ctx context.Context, from string, defaultAddr str
if err != nil {
logger.Error(
ctx, "could not parse address",
logger.E(errors.WithStack(err)), logger.F("address", addr.String()),
logger.CapturedE(errors.WithStack(err)), logger.F("address", addr.String()),
)
continue
@ -424,6 +415,13 @@ func newAppRepository(host string, basePort uint64, manifests ...*app.Manifest)
)
}
func initAppID(manifest *app.Manifest) ModuleDepFunc {
return func(deps *moduleDeps) error {
deps.AppID = manifest.ID
return nil
}
}
func initAppRepository(repo appModule.Repository) ModuleDepFunc {
return func(deps *moduleDeps) error {
deps.AppRepository = repo
@ -437,21 +435,32 @@ func initMemoryBus(deps *moduleDeps) error {
return nil
}
func initDatastores(storageFile string, appID app.ID) ModuleDepFunc {
func initDatastores(documentStoreDSN, blobStoreDSN, shareStoreDSN string, appID app.ID) ModuleDepFunc {
return func(deps *moduleDeps) error {
storageFile = injectAppID(storageFile, appID)
documentStoreDSN = injectAppID(documentStoreDSN, appID)
if err := ensureDir(storageFile); err != nil {
return errors.WithStack(err)
}
db, err := storageSqlite.Open(storageFile)
documentStore, err := driver.NewDocumentStore(documentStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.DocumentStore = storageSqlite.NewDocumentStoreWithDB(db)
deps.BlobStore = storageSqlite.NewBlobStoreWithDB(db)
deps.DocumentStore = documentStore
blobStoreDSN = injectAppID(blobStoreDSN, appID)
blobStore, err := driver.NewBlobStore(blobStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.BlobStore = blobStore
shareStore, err := driver.NewShareStore(shareStoreDSN)
if err != nil {
return errors.WithStack(err)
}
deps.ShareStore = shareStore
return nil
}
@ -471,17 +480,3 @@ func initAccounts(accountsFile string, appID app.ID) ModuleDepFunc {
return nil
}
}
func initShareRepository(shareRepositoryFile string) ModuleDepFunc {
return func(deps *moduleDeps) error {
if err := ensureDir(shareRepositoryFile); err != nil {
return errors.WithStack(err)
}
repo := shareSqlite.NewRepository(shareRepositoryFile)
deps.ShareRepository = repo
return nil
}
}

View File

@ -0,0 +1,75 @@
package auth
import (
"encoding/json"
"fmt"
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/flag"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
)
func CheckToken() *cli.Command {
return &cli.Command{
Name: "check-token",
Usage: "Validate and print the given token with the private key",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "token",
Required: true,
},
flag.PrivateKey,
flag.PrivateKeySigningAlgorithm,
flag.PrivateKeyDefaultSize,
},
Action: func(ctx *cli.Context) error {
privateKeyFile := flag.GetPrivateKey(ctx)
signingAlgorithm := flag.GetSigningAlgorithm(ctx)
privateKeyDefaultSize := flag.GetPrivateKeyDefaultSize(ctx)
rawToken := ctx.String("token")
if rawToken == "" {
return errors.New("you must provide a value for --token flag")
}
privateKey, err := jwtutil.LoadOrGenerateKey(
privateKeyFile,
privateKeyDefaultSize,
)
if err != nil {
return errors.WithStack(err)
}
keySet, err := jwtutil.NewKeySet()
if err != nil {
return errors.WithStack(err)
}
err = jwtutil.AddKeyWithSigningAlgo(keySet, privateKey, jwa.SignatureAlgorithm(signingAlgorithm))
if err != nil {
return errors.WithStack(err)
}
token, err := jwtutil.Parse([]byte(rawToken), keySet)
if err != nil {
return errors.WithStack(err)
}
claims, err := token.AsMap(ctx.Context)
if err != nil {
return errors.WithStack(err)
}
json, err := json.MarshalIndent(claims, "", " ")
if err != nil {
return errors.WithStack(err)
}
fmt.Println(string(json))
return nil
},
}
}

View File

@ -0,0 +1,58 @@
package auth
import (
"fmt"
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/flag"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
)
func NewToken() *cli.Command {
return &cli.Command{
Name: "new-token",
Usage: "Generate new authentication token",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "tenant",
Required: true,
},
flag.PrivateKey,
flag.PrivateKeySigningAlgorithm,
flag.PrivateKeyDefaultSize,
},
Action: func(ctx *cli.Context) error {
privateKeyFile := flag.GetPrivateKey(ctx)
signingAlgorithm := flag.GetSigningAlgorithm(ctx)
privateKeyDefaultSize := flag.GetPrivateKeyDefaultSize(ctx)
tenant := ctx.String("tenant")
if tenant == "" {
return errors.New("you must provide a value for --tenant flag")
}
privateKey, err := jwtutil.LoadOrGenerateKey(
privateKeyFile,
privateKeyDefaultSize,
)
if err != nil {
return errors.WithStack(err)
}
claims := map[string]any{
"tenant": tenant,
}
token, err := jwtutil.SignedToken(privateKey, jwa.SignatureAlgorithm(signingAlgorithm), claims)
if err != nil {
return errors.Wrap(err, "could not generate signed token")
}
fmt.Println(string(token))
return nil
},
}
}

View File

@ -0,0 +1,16 @@
package auth
import (
"github.com/urfave/cli/v2"
)
func Root() *cli.Command {
return &cli.Command{
Name: "auth",
Usage: "Auth related command",
Subcommands: []*cli.Command{
NewToken(),
CheckToken(),
},
}
}

View File

@ -0,0 +1,43 @@
package flag
import (
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/urfave/cli/v2"
)
const PrivateKeyFlagName = "private-key"
var PrivateKey = &cli.StringFlag{
Name: PrivateKeyFlagName,
EnvVars: []string{"STORAGE_SERVER_PRIVATE_KEY"},
Value: "storage-server.key",
TakesFile: true,
}
func GetPrivateKey(ctx *cli.Context) string {
return ctx.String(PrivateKeyFlagName)
}
const SigningAlgorithmFlagName = "signing-algorithm"
var PrivateKeySigningAlgorithm = &cli.StringFlag{
Name: SigningAlgorithmFlagName,
EnvVars: []string{"STORAGE_SERVER_SIGNING_ALGORITHM"},
Value: jwa.RS256.String(),
}
func GetSigningAlgorithm(ctx *cli.Context) string {
return ctx.String(SigningAlgorithmFlagName)
}
const PrivateKeyDefaultSizeFlagName = "private-key-default-size"
var PrivateKeyDefaultSize = &cli.IntFlag{
Name: PrivateKeyDefaultSizeFlagName,
EnvVars: []string{"STORAGE_SERVER_PRIVATE_KEY_DEFAULT_SIZE"},
Value: 2048,
}
func GetPrivateKeyDefaultSize(ctx *cli.Context) int {
return ctx.Int(PrivateKeyDefaultSizeFlagName)
}

View File

@ -0,0 +1,48 @@
package command
import (
"context"
"fmt"
"os"
"sort"
"github.com/urfave/cli/v2"
)
func Main(commands ...*cli.Command) {
ctx := context.Background()
app := &cli.App{
Name: "storage-server",
Usage: "Edge storage server",
Commands: commands,
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "debug",
EnvVars: []string{"DEBUG"},
Value: false,
},
},
}
app.ExitErrHandler = func(ctx *cli.Context, err error) {
if err == nil {
return
}
debug := ctx.Bool("debug")
if !debug {
fmt.Printf("[ERROR] %v\n", err)
} else {
fmt.Printf("%+v", err)
}
}
sort.Sort(cli.FlagsByName(app.Flags))
sort.Sort(cli.CommandsByName(app.Commands))
if err := app.RunContext(ctx, os.Args); err != nil {
os.Exit(1)
}
}

View File

@ -0,0 +1,331 @@
package command
import (
"context"
"fmt"
"net/http"
"os"
"strings"
"sync"
"time"
"github.com/getsentry/sentry-go"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/keegancsmith/rpc"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"gitlab.com/wpetit/goweb/logger"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
// Register storage drivers
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/flag"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func Run() *cli.Command {
return &cli.Command{
Name: "run",
Usage: "Run server",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "address",
EnvVars: []string{"STORAGE_SERVER_ADDRESS"},
Aliases: []string{"addr"},
Value: ":3001",
},
&cli.IntFlag{
Name: "log-level",
EnvVars: []string{"STORAGE_SERVER_LOG_LEVEL"},
Value: int(logger.LevelError),
},
&cli.StringFlag{
Name: "blobstore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_BLOBSTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
},
&cli.StringFlag{
Name: "documentstore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_DOCUMENTSTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
},
&cli.StringFlag{
Name: "sharestore-dsn-pattern",
EnvVars: []string{"STORAGE_SERVER_SHARESTORE_DSN_PATTERN"},
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
},
&cli.StringFlag{
Name: "sentry-dsn",
EnvVars: []string{"STORAGE_SERVER_SENTRY_DSN"},
Value: "",
},
&cli.StringFlag{
Name: "sentry-environment",
EnvVars: []string{"STORAGE_SERVER_SENTRY_ENVIRONMENT"},
Value: "",
},
flag.PrivateKey,
flag.PrivateKeySigningAlgorithm,
flag.PrivateKeyDefaultSize,
&cli.DurationFlag{
Name: "cache-ttl",
EnvVars: []string{"STORAGE_SERVER_CACHE_TTL"},
Value: time.Hour,
},
&cli.IntFlag{
Name: "cache-size",
EnvVars: []string{"STORAGE_SERVER_CACHE_SIZE"},
Value: 32,
},
},
Action: func(ctx *cli.Context) error {
addr := ctx.String("address")
blobStoreDSNPattern := ctx.String("blobstore-dsn-pattern")
documentStoreDSNPattern := ctx.String("documentstore-dsn-pattern")
shareStoreDSNPattern := ctx.String("sharestore-dsn-pattern")
cacheSize := ctx.Int("cache-size")
cacheTTL := ctx.Duration("cache-ttl")
privateKeyFile := flag.GetPrivateKey(ctx)
signingAlgorithm := flag.GetSigningAlgorithm(ctx)
privateKeyDefaultSize := flag.GetPrivateKeyDefaultSize(ctx)
logLevel := ctx.Int("log-level")
logger.SetLevel(logger.Level(logLevel))
sentryDSN := ctx.String("sentry-dsn")
sentryEnvironment := ctx.String("sentry-environment")
if sentryDSN != "" {
if sentryEnvironment == "" {
sentryEnvironment, _ = os.Hostname()
}
err := sentry.Init(sentry.ClientOptions{
Dsn: sentryDSN,
Debug: logLevel == int(logger.LevelDebug),
AttachStacktrace: true,
Environment: sentryEnvironment,
})
if err != nil {
logger.Error(ctx.Context, "could not initialize sentry", logger.CapturedE(errors.WithStack(err)))
}
logger.SetCaptureFunc(func(err error) {
sentry.CaptureException(err)
})
defer sentry.Flush(2 * time.Second)
}
router := chi.NewRouter()
privateKey, err := jwtutil.LoadOrGenerateKey(
privateKeyFile,
privateKeyDefaultSize,
)
if err != nil {
return errors.WithStack(err)
}
getBlobStoreServer := createGetCachedStoreServer(
func(dsn string) (storage.BlobStore, error) {
return driver.NewBlobStore(dsn)
},
func(store storage.BlobStore) *rpc.Server {
return server.NewBlobStoreServer(store)
},
)
getShareStoreServer := createGetCachedStoreServer(
func(dsn string) (share.Store, error) {
return driver.NewShareStore(dsn)
},
func(store share.Store) *rpc.Server {
return server.NewShareStoreServer(store)
},
)
getDocumentStoreServer := createGetCachedStoreServer(
func(dsn string) (storage.DocumentStore, error) {
return driver.NewDocumentStore(dsn)
},
func(store storage.DocumentStore) *rpc.Server {
return server.NewDocumentStoreServer(store)
},
)
router.Use(middleware.RealIP)
router.Use(middleware.Logger)
logger.Debug(ctx.Context, "using authentication", logger.F("privateKey", privateKeyFile), logger.F("signingAlgorithm", signingAlgorithm))
router.Use(authenticate(privateKey, jwa.SignatureAlgorithm(signingAlgorithm)))
router.Handle("/blobstore", createStoreHandler(getBlobStoreServer, blobStoreDSNPattern, true, cacheSize, cacheTTL))
router.Handle("/documentstore", createStoreHandler(getDocumentStoreServer, documentStoreDSNPattern, true, cacheSize, cacheTTL))
router.Handle("/sharestore", createStoreHandler(getShareStoreServer, shareStoreDSNPattern, false, cacheSize, cacheTTL))
logger.Info(ctx.Context, "listening", logger.F("addr", addr))
if err := http.ListenAndServe(addr, router); err != nil {
return errors.WithStack(err)
}
return nil
},
}
}
type getRPCServerFunc func(cacheSize int, cacheTTL time.Duration, tenant, appID, dsnPattern string) (*rpc.Server, error)
func createGetCachedStoreServer[T any](storeFactory func(dsn string) (T, error), serverFactory func(store T) *rpc.Server) getRPCServerFunc {
var (
cache *expirable.LRU[string, *rpc.Server]
initCache sync.Once
)
return func(cacheSize int, cacheTTL time.Duration, tenant, appID, dsnPattern string) (*rpc.Server, error) {
initCache.Do(func() {
cache = expirable.NewLRU[string, *rpc.Server](cacheSize, nil, cacheTTL)
})
key := fmt.Sprintf("%s:%s", tenant, appID)
storeServer, _ := cache.Get(key)
if storeServer != nil {
return storeServer, nil
}
dsn := strings.ReplaceAll(dsnPattern, "%TENANT%", tenant)
dsn = strings.ReplaceAll(dsn, "%APPID%", appID)
store, err := storeFactory(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
storeServer = serverFactory(store)
cache.Add(key, storeServer)
return storeServer, nil
}
}
func createStoreHandler(getStoreServer getRPCServerFunc, dsnPattern string, appIDRequired bool, cacheSize int, cacheTTL time.Duration) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
tenant, ok := ctx.Value("tenant").(string)
if !ok || tenant == "" {
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
appID := r.URL.Query().Get("appId")
if appIDRequired && appID == "" {
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
}
server, err := getStoreServer(cacheSize, cacheTTL, tenant, appID, dsnPattern)
if err != nil {
logger.Error(r.Context(), "could not retrieve store server", logger.CapturedE(errors.WithStack(err)), logger.F("tenant", tenant))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
server.ServeHTTP(w, r)
})
}
func authenticate(privateKey jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) func(http.Handler) http.Handler {
var (
createKeySet sync.Once
err error
getKeySet jwtutil.GetKeySetFunc
)
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
createKeySet.Do(func() {
var keySet jwk.Set
keySet, err = jwtutil.NewKeySet()
if err != nil {
err = errors.WithStack(err)
return
}
err = jwtutil.AddKeyWithSigningAlgo(keySet, privateKey, signingAlgorithm)
if err != nil {
err = errors.WithStack(err)
return
}
getKeySet = func() (jwk.Set, error) {
return keySet, nil
}
})
if err != nil {
logger.Error(ctx, "could not create keyset accessor", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
token, err := jwtutil.FindToken(r, getKeySet, jwtutil.WithFinders(
jwtutil.FindTokenFromQueryString("token"),
))
if err != nil {
logger.Error(ctx, "could not find jwt token", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
tokenMap, err := token.AsMap(ctx)
if err != nil {
logger.Error(ctx, "could not transform token to map", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
rawTenant, exists := tokenMap["tenant"]
if !exists {
logger.Warn(ctx, "could not find tenant claim", logger.F("token", token))
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
tenant, ok := rawTenant.(string)
if !ok {
logger.Warn(ctx, "unexpected tenant claim value", logger.F("token", token))
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
return
}
r = r.WithContext(context.WithValue(ctx, "tenant", tenant))
next.ServeHTTP(w, r)
})
}
}

View File

@ -0,0 +1,13 @@
package main
import (
"forge.cadoles.com/arcad/edge/cmd/storage-server/command"
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/auth"
)
func main() {
command.Main(
command.Run(),
auth.Root(),
)
}

34
go.mod
View File

@ -1,27 +1,35 @@
module forge.cadoles.com/arcad/edge
go 1.19
go 1.21
require (
github.com/getsentry/sentry-go v0.25.0
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/hashicorp/mdns v1.0.5
github.com/keegancsmith/rpc v1.3.0
github.com/klauspost/compress v1.16.6
github.com/lestrrat-go/jwx/v2 v2.0.8
github.com/ulikunitz/xz v0.5.11
modernc.org/sqlite v1.20.4
)
require (
cloud.google.com/go v0.75.0 // indirect
github.com/allegro/bigcache/v3 v3.1.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/go-playground/locales v0.12.1 // indirect
github.com/go-playground/universal-translator v0.16.0 // indirect
github.com/go-playground/locales v0.14.0 // indirect
github.com/go-playground/universal-translator v0.18.0 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/gogo/protobuf v0.0.0-20161014173244-50d1bd39ce4e // indirect
github.com/leodido/go-urn v1.1.0 // indirect
github.com/jackc/puddle/v2 v2.2.1 // indirect
github.com/leodido/go-urn v1.2.1 // indirect
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/httprc v1.0.4 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.0 // indirect
github.com/miekg/dns v1.1.53 // indirect
golang.org/x/sync v0.1.0 // indirect
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 // indirect
google.golang.org/grpc v1.35.0 // indirect
gopkg.in/go-playground/validator.v9 v9.29.1 // indirect
@ -43,12 +51,12 @@ require (
github.com/go-chi/chi/v5 v5.0.8
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.4.2 // indirect
github.com/igm/sockjs-go/v3 v3.0.2
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/oklog/ulid/v2 v2.1.0
github.com/orcaman/concurrent-map v1.0.0
@ -57,14 +65,14 @@ require (
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/urfave/cli/v2 v2.24.3
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648
go.opencensus.io v0.22.5 // indirect
golang.org/x/crypto v0.7.0
golang.org/x/crypto v0.10.0
golang.org/x/mod v0.10.0
golang.org/x/net v0.9.0 // indirect
golang.org/x/sys v0.7.0 // indirect
golang.org/x/term v0.7.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/net v0.11.0
golang.org/x/sys v0.9.0 // indirect
golang.org/x/term v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect
golang.org/x/tools v0.8.0 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gopkg.in/yaml.v2 v2.4.0

95
go.sum
View File

@ -53,6 +53,8 @@ github.com/alecthomas/kong v0.2.1-0.20190708041108-0548c6b1afae/go.mod h1:+inYUS
github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MRgZdU3vrFd05IQ89AxUZ0aYdF39BYoNFa324SodPCA=
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkxI1zYWl1QLnEqAqEARBEYa8FQnQcY=
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/allegro/bigcache/v3 v3.1.0 h1:H2Vp8VOvxcrB91o86fUSVJFqeuz8kpyyB02eH3bSzwk=
github.com/allegro/bigcache/v3 v3.1.0/go.mod h1:aPyh7jEvrog9zAwx5N7+JUQX5dZTSGpxF1LAR4dr35I=
github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692 h1:JW4WZlqyaNWUUahfr7MigeDW6jmtam5cTzzo1lwsFhE=
github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692/go.mod h1:Au0ipPuCBA7zsOC61SnyrYetm8VT3vo1UJtwHeYke44=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -101,16 +103,20 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI=
github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
@ -143,8 +149,10 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@ -157,7 +165,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -171,8 +181,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -188,6 +198,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0=
github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE=
@ -196,12 +208,18 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/igm/sockjs-go/v3 v3.0.2 h1:2m0k53w0DBiGozeQUIEPR6snZFmpFpYvVsGnfLPNXbE=
github.com/igm/sockjs-go/v3 v3.0.2/go.mod h1:UqchsOjeagIBFHvd+RZpLaVRbCwGilEC08EDHsD1jYE=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk=
github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@ -210,8 +228,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
@ -225,14 +243,17 @@ github.com/lestrrat-go/jwx/v2 v2.0.8/go.mod h1:zLxnyv9rTlEvOUHbc48FAfIL8iYu2hHvI
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw=
@ -247,6 +268,9 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD
github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -258,8 +282,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
@ -275,8 +300,11 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0=
github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
@ -289,8 +317,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b h1:nkvOl8TCj/mErADnwFFynjxBtC+hHsrESw6rw56JGmg=
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b/go.mod h1:3sus4zjoUv1GB7eDLL60QaPkUnXJCWBpjvbe0jWifeY=
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648 h1:t2UQmCmUoElIBBuVTqxqo8DcTJA/exQ/Q7XycfLqCZo=
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648/go.mod h1:WdxGjM3HJWgBkUa4TwaTXUqY2BnRKlNSyUIv1aF4jxk=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@ -306,8 +334,9 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -341,6 +370,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20161013035702-8b4af36cd21a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -379,8 +409,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -401,6 +433,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -442,13 +475,17 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -458,8 +495,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@ -508,6 +547,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@ -601,8 +641,11 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@ -632,7 +675,9 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
@ -646,9 +691,11 @@ modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE=
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

View File

@ -24,6 +24,7 @@
mocha.checkLeaks();
</script>
<script src="/edge/sdk/client.js"></script>
<script src="/test/util.js"></script>
<script src="/test/client-sdk.js"></script>
<script src="/test/auth-module.js"></script>
<script src="/test/net-module.js"></script>
@ -31,6 +32,7 @@
<script src="/test/file-module.js"></script>
<script src="/test/app-module.js"></script>
<script src="/test/fetch-module.js"></script>
<script src="/test/share-module.js"></script>
<script class="mocha-exec">
mocha.run();
@ -44,6 +46,7 @@
.setItem('file-module', 'File Module', { linkUrl: '/?grep=File%20Module', order: 6})
.setItem('app-module', 'App Module', { linkUrl: '/?grep=App%20Module' , order: 7})
.setItem('fetch-module', 'Fetch Module', { linkUrl: '/?grep=Fetch%20Module' , order: 8})
.setItem('share-module', 'Share Module', { linkUrl: '/?grep=Share%20Module' , order: 9})
</script>
</body>
</html>

View File

@ -0,0 +1,29 @@
describe('Share Module', function() {
before(() => {
return Edge.Client.connect();
});
after(() => {
Edge.Client.disconnect();
});
it('should create a new resource and find it', async () => {
const resource = await TestUtil.serverSideCall('share', 'upsertResource', 'my-resource', { name: "color", type: "text", value: "red" });
chai.assert.isNotNull(resource);
chai.assert.equal(resource.origin, 'edge.sdk.client.test')
const results = await TestUtil.serverSideCall('share', 'findResources', 'color', 'text');
chai.assert.isAbove(results.length, 0);
const createdResource = results.find(res => {
return res.origin === 'edge.sdk.client.test' &&
res.attributes.find(attr => attr.name === 'color' && attr.type === 'text')
})
chai.assert.isNotNull(createdResource)
console.log(createdResource)
});
});

View File

@ -0,0 +1,7 @@
(function(TestUtil) {
TestUtil.serverSideCall = (module, func, ...args) => {
return Edge.Client.rpc('serverSideCall', { module, func, args })
}
console.log(TestUtil)
}(globalThis.TestUtil = globalThis.TestUtil || {}));

View File

@ -15,6 +15,8 @@ function onInit() {
rpc.register("listApps");
rpc.register("getApp");
rpc.register("getAppUrl");
rpc.register("serverSideCall", serverSideCall)
}
// Called for each client message
@ -103,4 +105,9 @@ function getAppUrl(ctx, params) {
function onClientFetch(ctx, url, remoteAddr) {
return { allow: url === 'http://example.com' };
}
function serverSideCall(ctx, params) {
console.log("Calling %s.%s(args...)", params.module, params.func)
return globalThis[params.module][params.func].call(null, ctx, ...params.args);
}

View File

@ -4,7 +4,7 @@ ARG HTTP_PROXY=
ARG HTTPS_PROXY=
ARG http_proxy=
ARG https_proxy=
ARG GO_VERSION=1.20.2
ARG GO_VERSION=1.21.2
# Install dev environment dependencies
RUN export DEBIAN_FRONTEND=noninteractive &&\

View File

@ -0,0 +1,75 @@
#!/bin/sh
use_systemctl="True"
systemd_version=0
if ! command -V systemctl >/dev/null 2>&1; then
use_systemctl="False"
else
systemd_version=$(systemctl --version | head -1 | cut -d ' ' -f 2)
fi
service_name=storage-server
cleanup() {
if [ "${use_systemctl}" = "False" ]; then
rm -f /usr/lib/systemd/system/${service_name}.service
else
rm -f /etc/chkconfig/${service_name}
rm -f /etc/init.d/${service_name}
fi
}
cleanInstall() {
printf "\033[32m Post Install of an clean install\033[0m\n"
if [ "${use_systemctl}" = "False" ]; then
if command -V chkconfig >/dev/null 2>&1; then
chkconfig --add ${service_name}
fi
service ${service_name} restart || :
else
if [[ "${systemd_version}" -lt 231 ]]; then
printf "\033[31m systemd version %s is less then 231, fixing the service file \033[0m\n" "${systemd_version}"
sed -i "s/=+/=/g" /usr/lib/systemd/system/${service_name}.service
fi
printf "\033[32m Reload the service unit from disk\033[0m\n"
systemctl daemon-reload || :
printf "\033[32m Unmask the service\033[0m\n"
systemctl unmask ${service_name} || :
printf "\033[32m Set the preset flag for the service unit\033[0m\n"
systemctl preset ${service_name} || :
printf "\033[32m Set the enabled flag for the service unit\033[0m\n"
systemctl enable ${service_name} || :
systemctl restart ${service_name} || :
fi
}
upgrade() {
printf "\033[32m Post Install of an upgrade\033[0m\n"
systemctl daemon-reload || :
systemctl restart ${service_name} || :
}
# Step 2, check if this is a clean install or an upgrade
action="$1"
if [ "$1" = "configure" ] && [ -z "$2" ]; then
action="install"
elif [ "$1" = "configure" ] && [ -n "$2" ]; then
action="upgrade"
fi
case "$action" in
"1" | "install")
cleanInstall
;;
"2" | "upgrade")
printf "\033[32m Post Install of an upgrade\033[0m\n"
upgrade
;;
*)
printf "\033[32m Alpine\033[0m"
cleanInstall
;;
esac
cleanup

View File

@ -0,0 +1,9 @@
export STORAGE_SERVER_ADDRESS=:3001
export STORAGE_SERVER_BLOBSTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/%APPID%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
export STORAGE_SERVER_DOCUMENTSTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/%APPID%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
export STORAGE_SERVER_SHARESTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
export STORAGE_SERVER_PRIVATE_KEY="/var/lib/storage-server/storage-server.key"
export STORAGE_SERVER_PRIVATE_KEY_DEFAULT_SIZE="2048"
export STORAGE_SERVER_SIGNING_ALGORITHM="RS256"
export STORAGE_SERVER_CACHE_TTL=1h
export STORAGE_SERVER_CACHE_SIZE=32

View File

@ -0,0 +1,11 @@
#!/sbin/openrc-run
command="/usr/bin/storage-server"
command_args="run"
supervisor=supervise-daemon
output_log="/var/log/storage-server.log"
error_log="$output_log"
depend() {
need net
}

View File

@ -0,0 +1,9 @@
STORAGE_SERVER_ADDRESS=:3001
STORAGE_SERVER_BLOBSTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/%APPID%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
STORAGE_SERVER_DOCUMENTSTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/%APPID%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
STORAGE_SERVER_SHARESTORE_DSN_PATTERN="sqlite:///var/lib/storage-server/data/%TENANT%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000"
STORAGE_SERVER_PRIVATE_KEY="/var/lib/storage-server/storage-server.key"
STORAGE_SERVER_PRIVATE_KEY_DEFAULT_SIZE="2048"
STORAGE_SERVER_SIGNING_ALGORITHM="RS256"
STORAGE_SERVER_CACHE_TTL=1h
STORAGE_SERVER_CACHE_SIZE=32

View File

@ -0,0 +1,35 @@
[Unit]
Description=storage-server service
After=network.target
[Service]
Type=simple
Restart=on-failure
EnvironmentFile=/etc/storage-server/environ
ExecStart=/usr/bin/storage-server run
EnvironmentFile=/etc/storage-server/environ
NoNewPrivileges=yes
PrivateTmp=yes
PrivateDevices=yes
PrivateUsers=yes
DynamicUser=yes
StateDirectory=storage-server
DevicePolicy=closed
ProtectSystem=true
ProtectHome=read-only
ProtectKernelLogs=yes
ProtectProc=invisible
ProtectClock=yes
ProtectControlGroups=yes
ProtectKernelModules=yes
ProtectKernelTunables=yes
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
RestrictNamespaces=yes
RestrictRealtime=yes
RestrictSUIDSGID=yes
MemoryDenyWriteExecute=yes
LockPersonality=yes
CapabilityBoundingSet=~CAP_SETUID CAP_SETGID CAP_SETPCAP CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_CHOWN CAP_FSETID CAP_SETFCAP CAP_DAC_OVERRIDE CAP_DAC_READ_SEARCH CAP_FOWNER CAP_IPC_OWNER CAP_NET_ADMIN CAP_WAKE_ALARM CAP_SYS_TTY_CONFIG
[Install]
WantedBy=multi-user.target

View File

@ -2,15 +2,20 @@
**/*.tmpl
pkg/sdk/client/src/**/*.js
pkg/sdk/client/src/**/*.ts
misc/client-sdk-testsuite/src/**/*
misc/client-sdk-testsuite/dist/server/*.js
modd.conf
.env
{
prep: make build-sdk
prep: make build-client-sdk-test-app
prep: make build
prep: make build-sdk build-cli build-storage-server
daemon: make run-app
daemon: make run-storage-server
}
misc/client-sdk-testsuite/src/**/*
{
prep: make build-client-sdk-test-app
}
**/*.go {
prep: make GOTEST_ARGS="-short" test
# prep: make GOTEST_ARGS="-short" test
}

36
pkg/app/option.go Normal file
View File

@ -0,0 +1,36 @@
package app
import (
"context"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type Options struct {
ModuleFactories []ServerModuleFactory
ErrorHandler func(ctx context.Context, err error)
}
type OptionFunc func(opts *Options)
func NewOptions(funcs ...OptionFunc) *Options {
opts := &Options{
ModuleFactories: make([]ServerModuleFactory, 0),
ErrorHandler: func(ctx context.Context, err error) {
logger.Error(ctx, err.Error(), logger.E(errors.WithStack(err)))
},
}
for _, fn := range funcs {
fn(opts)
}
return opts
}
func WithModulesFactories(factories ...ServerModuleFactory) OptionFunc {
return func(opts *Options) {
opts.ModuleFactories = factories
}
}

View File

@ -13,7 +13,7 @@ import (
var (
ErrFuncDoesNotExist = errors.New("function does not exist")
ErUnknownError = errors.New("unknown error")
ErrUnknownError = errors.New("unknown error")
)
type Server struct {
@ -88,9 +88,9 @@ func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...inter
if recovered := recover(); recovered != nil {
revoveredErr, ok := recovered.(error)
if ok {
logger.Error(ctx, "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
logger.Error(ctx, "recovered runtime error", logger.CapturedE(errors.WithStack(revoveredErr)))
err = errors.WithStack(ErUnknownError)
err = errors.WithStack(ErrUnknownError)
return
}
@ -162,7 +162,7 @@ func (s *Server) WaitForPromise(promise *goja.Promise) goja.Value {
return value
}
func (s *Server) Start() error {
func (s *Server) Start(ctx context.Context) error {
s.loop.Start()
var err error
@ -171,7 +171,7 @@ func (s *Server) Start() error {
rt.SetFieldNameMapper(goja.TagFieldNameMapper("goja", true))
rt.SetRandSource(createRandomSource())
if err = s.initModules(rt); err != nil {
if err = s.initModules(ctx, rt); err != nil {
err = errors.WithStack(err)
}
})
@ -186,7 +186,7 @@ func (s *Server) Stop() {
s.loop.Stop()
}
func (s *Server) initModules(rt *goja.Runtime) error {
func (s *Server) initModules(ctx context.Context, rt *goja.Runtime) error {
modules := make([]ServerModule, 0, len(s.factories))
for _, moduleFactory := range s.factories {
@ -206,9 +206,9 @@ func (s *Server) initModules(rt *goja.Runtime) error {
continue
}
logger.Debug(context.Background(), "initializing module", logger.F("module", initMod.Name()))
logger.Debug(ctx, "initializing module", logger.F("module", initMod.Name()))
if err := initMod.OnInit(rt); err != nil {
if err := initMod.OnInit(ctx, rt); err != nil {
return errors.WithStack(err)
}
}

View File

@ -1,6 +1,8 @@
package app
import (
"context"
"github.com/dop251/goja"
)
@ -13,5 +15,5 @@ type ServerModule interface {
type InitializableModule interface {
ServerModule
OnInit(rt *goja.Runtime) error
OnInit(ctx context.Context, rt *goja.Runtime) error
}

View File

@ -3,7 +3,7 @@ package bundle
import (
"bytes"
"context"
"io/ioutil"
"io"
"net/http"
"os"
"path"
@ -40,8 +40,6 @@ func (fs *FileSystem) Open(name string) (http.File, error) {
return nil, err
}
logger.Error(ctx, "could not open bundle file", logger.E(err))
return nil, errors.Wrapf(err, "could not open bundle file '%s'", p)
}
defer readCloser.Close()
@ -53,16 +51,14 @@ func (fs *FileSystem) Open(name string) (http.File, error) {
if fileInfo.IsDir() {
files, err := fs.bundle.Dir(p)
if err != nil {
logger.Error(ctx, "could not read bundle directory", logger.E(err))
return nil, errors.Wrapf(err, "could not read bundle directory '%s'", p)
}
file.files = files
} else {
data, err := ioutil.ReadAll(readCloser)
data, err := io.ReadAll(readCloser)
if err != nil {
logger.Error(ctx, "could not read bundle file", logger.E(err))
logger.Error(ctx, "could not read bundle file", logger.CapturedE(errors.WithStack(err)))
return nil, errors.Wrapf(err, "could not read bundle file '%s'", p)
}

View File

@ -13,6 +13,7 @@ type ArchiveExt string
const (
ExtZip ArchiveExt = "zip"
ExtTarGz ArchiveExt = "tar.gz"
ExtZim ArchiveExt = "zim"
)
func FromPath(path string) (Bundle, error) {
@ -56,5 +57,14 @@ func matchArchivePattern(archivePath string) (Bundle, error) {
return NewZipBundle(archivePath), nil
}
matches, err = filepath.Match(fmt.Sprintf("*.%s", ExtZim), base)
if err != nil {
return nil, errors.Wrapf(err, "could not match file archive '%s'", archivePath)
}
if matches {
return NewZimBundle(archivePath), nil
}
return nil, errors.WithStack(ErrUnknownBundleArchiveExt)
}

View File

@ -0,0 +1,8 @@
package zim
import "io"
type BlobReader interface {
io.ReadCloser
Size() (int64, error)
}

View File

@ -0,0 +1,163 @@
package zim
import (
"bytes"
"encoding/binary"
"io"
"os"
"sync"
"github.com/pkg/errors"
)
type CompressedBlobReader struct {
reader *Reader
decoderFactory BlobDecoderFactory
clusterStartOffset uint64
clusterEndOffset uint64
blobIndex uint32
blobSize int
readOffset uint64
loadCluster sync.Once
loadClusterErr error
data []byte
closed bool
}
// Size implements BlobReader.
func (r *CompressedBlobReader) Size() (int64, error) {
if err := r.loadClusterData(); err != nil {
return 0, errors.WithStack(err)
}
return int64(len(r.data)), nil
}
// Close implements io.ReadCloser.
func (r *CompressedBlobReader) Close() error {
clear(r.data)
r.closed = true
return nil
}
// Read implements io.ReadCloser.
func (r *CompressedBlobReader) Read(p []byte) (int, error) {
if err := r.loadClusterData(); err != nil {
return 0, errors.WithStack(err)
}
length := len(p)
remaining := len(r.data) - int(r.readOffset)
if length > remaining {
length = remaining
}
chunk := make([]byte, length)
copy(chunk, r.data[r.readOffset:int(r.readOffset)+length])
copy(p, chunk)
if length == remaining {
return length, io.EOF
}
r.readOffset += uint64(length)
return length, nil
}
func (r *CompressedBlobReader) loadClusterData() error {
if r.closed {
return errors.WithStack(os.ErrClosed)
}
r.loadCluster.Do(func() {
compressedData := make([]byte, r.clusterEndOffset-r.clusterStartOffset)
if err := r.reader.readRange(int64(r.clusterStartOffset+1), compressedData); err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
blobBuffer := bytes.NewBuffer(compressedData)
decoder, err := r.decoderFactory(blobBuffer)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
defer decoder.Close()
uncompressedData, err := io.ReadAll(decoder)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
var (
blobStart uint64
blobEnd uint64
)
if r.blobSize == 8 {
blobStart64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
blobStart = blobStart64
blobEnd64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
blobEnd = blobEnd64
} else {
blobStart32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
blobStart = uint64(blobStart32)
blobEnd32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
if err != nil {
r.loadClusterErr = errors.WithStack(err)
return
}
blobEnd = uint64(blobEnd32)
}
r.data = make([]byte, blobEnd-blobStart)
copy(r.data, uncompressedData[blobStart:blobEnd])
})
if r.loadClusterErr != nil {
return errors.WithStack(r.loadClusterErr)
}
return nil
}
type BlobDecoderFactory func(io.Reader) (io.ReadCloser, error)
func NewCompressedBlobReader(reader *Reader, decoderFactory BlobDecoderFactory, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
return &CompressedBlobReader{
reader: reader,
decoderFactory: decoderFactory,
clusterStartOffset: clusterStartOffset,
clusterEndOffset: clusterEndOffset,
blobIndex: blobIndex,
blobSize: blobSize,
readOffset: 0,
}
}
var _ BlobReader = &UncompressedBlobReader{}

View File

@ -0,0 +1,193 @@
package zim
import (
"encoding/binary"
"github.com/pkg/errors"
)
type zimCompression int
const (
zimCompressionNoneZeno zimCompression = 0
zimCompressionNone zimCompression = 1
zimCompressionNoneZLib zimCompression = 2
zimCompressionNoneBZip2 zimCompression = 3
zimCompressionNoneXZ zimCompression = 4
zimCompressionNoneZStandard zimCompression = 5
)
type ContentEntry struct {
*BaseEntry
mimeType string
clusterIndex uint32
blobIndex uint32
}
func (e *ContentEntry) Compression() (int, error) {
clusterHeader, _, _, err := e.readClusterInfo()
if err != nil {
return 0, errors.WithStack(err)
}
return int((clusterHeader << 4) >> 4), nil
}
func (e *ContentEntry) MimeType() string {
return e.mimeType
}
func (e *ContentEntry) Reader() (BlobReader, error) {
clusterHeader, clusterStartOffset, clusterEndOffset, err := e.readClusterInfo()
if err != nil {
return nil, errors.WithStack(err)
}
compression := (clusterHeader << 4) >> 4
extended := (clusterHeader<<3)>>7 == 1
blobSize := 4
if extended {
blobSize = 8
}
switch compression {
// Uncompressed blobs
case uint8(zimCompressionNoneZeno):
fallthrough
case uint8(zimCompressionNone):
startPos := clusterStartOffset + 1
blobOffset := uint64(e.blobIndex * uint32(blobSize))
data := make([]byte, 2*blobSize)
if err := e.reader.readRange(int64(startPos+blobOffset), data); err != nil {
return nil, errors.WithStack(err)
}
var (
blobStart uint64
blobEnd uint64
)
if extended {
blobStart64, err := readUint64(data[0:blobSize], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
blobStart = blobStart64
blobEnd64, err := readUint64(data[blobSize:blobSize*2], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
blobEnd = uint64(blobEnd64)
} else {
blobStart32, err := readUint32(data[0:blobSize], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
blobStart = uint64(blobStart32)
blobEnd32, err := readUint32(data[blobSize:blobSize*2], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
blobEnd = uint64(blobEnd32)
}
return NewUncompressedBlobReader(e.reader, startPos+blobStart, startPos+blobEnd, blobSize), nil
// Supported compression algorithms
case uint8(zimCompressionNoneXZ):
return NewXZBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
case uint8(zimCompressionNoneZStandard):
return NewZStdBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
// Unsupported compression algorithms
case uint8(zimCompressionNoneZLib):
fallthrough
case uint8(zimCompressionNoneBZip2):
fallthrough
default:
return nil, errors.Wrapf(ErrCompressionAlgorithmNotSupported, "unexpected compression algorithm '%d'", compression)
}
}
func (e *ContentEntry) Redirect() (*ContentEntry, error) {
return e, nil
}
func (e *ContentEntry) readClusterInfo() (uint8, uint64, uint64, error) {
startClusterOffset, clusterEndOffset, err := e.reader.getClusterOffsets(int(e.clusterIndex))
if err != nil {
return 0, 0, 0, errors.WithStack(err)
}
data := make([]byte, 1)
if err := e.reader.readRange(int64(startClusterOffset), data); err != nil {
return 0, 0, 0, errors.WithStack(err)
}
clusterHeader := uint8(data[0])
return clusterHeader, startClusterOffset, clusterEndOffset, nil
}
func (r *Reader) parseContentEntry(offset int64, base *BaseEntry) (*ContentEntry, error) {
entry := &ContentEntry{
BaseEntry: base,
}
data := make([]byte, 16)
if err := r.readRange(offset, data); err != nil {
return nil, errors.WithStack(err)
}
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
if mimeTypeIndex >= uint16(len(r.mimeTypes)) {
return nil, errors.Errorf("mime type index '%d' greater than mime types length '%d'", mimeTypeIndex, len(r.mimeTypes))
}
entry.mimeType = r.mimeTypes[mimeTypeIndex]
entry.namespace = Namespace(data[3:4])
clusterIndex, err := readUint32(data[8:12], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
entry.clusterIndex = clusterIndex
blobIndex, err := readUint32(data[12:16], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
entry.blobIndex = blobIndex
strs, _, err := r.readStringsAt(offset+16, 2, 1024)
if err != nil {
return nil, errors.WithStack(err)
}
if len(strs) > 0 {
entry.url = strs[0]
}
if len(strs) > 1 {
entry.title = strs[1]
}
return entry, nil
}

135
pkg/bundle/zim/entry.go Normal file
View File

@ -0,0 +1,135 @@
package zim
import (
"encoding/binary"
"fmt"
"github.com/pkg/errors"
)
type Entry interface {
Redirect() (*ContentEntry, error)
Namespace() Namespace
URL() string
FullURL() string
Title() string
}
type BaseEntry struct {
mimeTypeIndex uint16
namespace Namespace
url string
title string
reader *Reader
}
func (e *BaseEntry) Namespace() Namespace {
return e.namespace
}
func (e *BaseEntry) Title() string {
if e.title == "" {
return e.url
}
return e.title
}
func (e *BaseEntry) URL() string {
return e.url
}
func (e *BaseEntry) FullURL() string {
return toFullURL(e.Namespace(), e.URL())
}
func (r *Reader) parseBaseEntry(offset int64) (*BaseEntry, error) {
entry := &BaseEntry{
reader: r,
}
data := make([]byte, 3)
if err := r.readRange(offset, data); err != nil {
return nil, errors.WithStack(err)
}
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
entry.mimeTypeIndex = mimeTypeIndex
entry.namespace = Namespace(data[2])
return entry, nil
}
type RedirectEntry struct {
*BaseEntry
redirectIndex uint32
}
func (e *RedirectEntry) Redirect() (*ContentEntry, error) {
if e.redirectIndex >= uint32(len(e.reader.urlIndex)) {
return nil, errors.Wrapf(ErrInvalidIndex, "entry index '%d' out of bounds", e.redirectIndex)
}
entryPtr := e.reader.urlIndex[e.redirectIndex]
entry, err := e.reader.parseEntryAt(int64(entryPtr))
if err != nil {
return nil, errors.WithStack(err)
}
entry, err = entry.Redirect()
if err != nil {
return nil, errors.WithStack(err)
}
contentEntry, ok := entry.(*ContentEntry)
if !ok {
return nil, errors.WithStack(ErrInvalidRedirect)
}
return contentEntry, nil
}
func (r *Reader) parseRedirectEntry(offset int64, base *BaseEntry) (*RedirectEntry, error) {
entry := &RedirectEntry{
BaseEntry: base,
}
data := make([]byte, 4)
if err := r.readRange(offset+8, data); err != nil {
return nil, errors.WithStack(err)
}
redirectIndex, err := readUint32(data, binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
entry.redirectIndex = redirectIndex
strs, _, err := r.readStringsAt(offset+12, 2, 1024)
if err != nil {
return nil, errors.WithStack(err)
}
if len(strs) > 0 {
entry.url = strs[0]
}
if len(strs) > 1 {
entry.title = strs[1]
}
return entry, nil
}
func toFullURL(ns Namespace, url string) string {
if ns == "\x00" {
return url
}
return fmt.Sprintf("%s/%s", ns, url)
}

View File

@ -0,0 +1,46 @@
package zim
import "github.com/pkg/errors"
type EntryIterator struct {
index int
entry Entry
err error
reader *Reader
}
func (it *EntryIterator) Next() bool {
if it.err != nil {
return false
}
entryCount := it.reader.EntryCount()
if it.index >= int(entryCount-1) {
return false
}
entry, err := it.reader.EntryAt(it.index)
if err != nil {
it.err = errors.WithStack(err)
return false
}
it.entry = entry
it.index++
return true
}
func (it *EntryIterator) Err() error {
return it.err
}
func (it *EntryIterator) Index() int {
return it.index - 1
}
func (it *EntryIterator) Entry() Entry {
return it.entry
}

10
pkg/bundle/zim/error.go Normal file
View File

@ -0,0 +1,10 @@
package zim
import "errors"
var (
ErrInvalidIndex = errors.New("invalid index")
ErrNotFound = errors.New("not found")
ErrInvalidRedirect = errors.New("invalid redirect")
ErrCompressionAlgorithmNotSupported = errors.New("compression algorithm not supported")
)

66
pkg/bundle/zim/favicon.go Normal file
View File

@ -0,0 +1,66 @@
package zim
import "github.com/pkg/errors"
func (r *Reader) Favicon() (*ContentEntry, error) {
illustration, err := r.getMetadataIllustration()
if err != nil && !errors.Is(err, ErrNotFound) {
return nil, errors.WithStack(err)
}
if illustration != nil {
return illustration, nil
}
namespaces := []Namespace{V5NamespaceLayout, V5NamespaceImageFile}
urls := []string{"favicon", "favicon.png"}
for _, ns := range namespaces {
for _, url := range urls {
entry, err := r.EntryWithURL(ns, url)
if err != nil && !errors.Is(err, ErrNotFound) {
return nil, errors.WithStack(err)
}
if errors.Is(err, ErrNotFound) {
continue
}
content, err := entry.Redirect()
if err != nil {
return nil, errors.WithStack(err)
}
return content, nil
}
}
return nil, errors.WithStack(ErrNotFound)
}
func (r *Reader) getMetadataIllustration() (*ContentEntry, error) {
keys := []MetadataKey{MetadataIllustration96x96at2, MetadataIllustration48x48at1}
metadata, err := r.Metadata(keys...)
if err != nil {
return nil, errors.WithStack(err)
}
for _, k := range keys {
if _, exists := metadata[k]; exists {
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(k))
if err != nil {
return nil, errors.WithStack(err)
}
content, err := entry.Redirect()
if err != nil {
return nil, errors.WithStack(err)
}
return content, nil
}
}
return nil, errors.WithStack(ErrNotFound)
}

View File

@ -0,0 +1,81 @@
package zim
import (
"io"
"github.com/pkg/errors"
)
type MetadataKey string
// See https://wiki.openzim.org/wiki/Metadata
const (
MetadataName MetadataKey = "Name"
MetadataTitle MetadataKey = "Title"
MetadataDescription MetadataKey = "Description"
MetadataLongDescription MetadataKey = "LongDescription"
MetadataCreator MetadataKey = "Creator"
MetadataTags MetadataKey = "Tags"
MetadataDate MetadataKey = "Date"
MetadataPublisher MetadataKey = "Publisher"
MetadataFlavour MetadataKey = "Flavour"
MetadataSource MetadataKey = "Source"
MetadataLanguage MetadataKey = "Language"
MetadataIllustration48x48at1 MetadataKey = "Illustration_48x48@1"
MetadataIllustration96x96at2 MetadataKey = "Illustration_96x96@2"
)
var knownKeys = []MetadataKey{
MetadataName,
MetadataTitle,
MetadataDescription,
MetadataLongDescription,
MetadataCreator,
MetadataPublisher,
MetadataLanguage,
MetadataTags,
MetadataDate,
MetadataFlavour,
MetadataSource,
MetadataIllustration48x48at1,
MetadataIllustration96x96at2,
}
// Metadata returns a copy of the internal metadata map of the ZIM file.
func (r *Reader) Metadata(keys ...MetadataKey) (map[MetadataKey]string, error) {
if len(keys) == 0 {
keys = knownKeys
}
metadata := make(map[MetadataKey]string)
for _, key := range keys {
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(key))
if err != nil {
if errors.Is(err, ErrNotFound) {
continue
}
return nil, errors.WithStack(err)
}
content, err := entry.Redirect()
if err != nil {
return nil, errors.WithStack(err)
}
reader, err := content.Reader()
if err != nil {
return nil, errors.WithStack(err)
}
data, err := io.ReadAll(reader)
if err != nil {
return nil, errors.WithStack(err)
}
metadata[key] = string(data)
}
return metadata, nil
}

View File

@ -0,0 +1,23 @@
package zim
type Namespace string
const (
V6NamespaceContent Namespace = "C"
V6NamespaceMetadata Namespace = "M"
V6NamespaceWellKnown Namespace = "W"
V6NamespaceSearch Namespace = "X"
)
const (
V5NamespaceLayout Namespace = "-"
V5NamespaceArticle Namespace = "A"
V5NamespaceArticleMetadata Namespace = "B"
V5NamespaceImageFile Namespace = "I"
V5NamespaceImageText Namespace = "J"
V5NamespaceMetadata Namespace = "M"
V5NamespaceCategoryText Namespace = "U"
V5NamespaceCategoryArticleList Namespace = "V"
V5NamespaceCategoryPerArticle Namespace = "W"
V5NamespaceSearch Namespace = "X"
)

30
pkg/bundle/zim/option.go Normal file
View File

@ -0,0 +1,30 @@
package zim
import "time"
type Options struct {
URLCacheSize int
URLCacheTTL time.Duration
CacheSize int
}
type OptionFunc func(opts *Options)
func NewOptions(funcs ...OptionFunc) *Options {
funcs = append([]OptionFunc{
WithCacheSize(2048),
}, funcs...)
opts := &Options{}
for _, fn := range funcs {
fn(opts)
}
return opts
}
func WithCacheSize(size int) OptionFunc {
return func(opts *Options) {
opts.CacheSize = size
}
}

558
pkg/bundle/zim/reader.go Normal file
View File

@ -0,0 +1,558 @@
package zim
import (
"context"
"encoding/binary"
"fmt"
"io"
"os"
"strings"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
const zimFormatMagicNumber uint32 = 0x44D495A
const nullByte = '\x00'
const zimRedirect = 0xffff
type Reader struct {
majorVersion uint16
minorVersion uint16
uuid string
entryCount uint32
clusterCount uint32
urlPtrPos uint64
titlePtrPos uint64
clusterPtrPos uint64
mimeListPos uint64
mainPage uint32
layoutPage uint32
checksumPos uint64
mimeTypes []string
urlIndex []uint64
clusterIndex []uint64
cache *lru.Cache[string, Entry]
urls map[string]int
rangeReader RangeReadCloser
}
func (r *Reader) Version() (majorVersion, minorVersion uint16) {
return r.majorVersion, r.minorVersion
}
func (r *Reader) EntryCount() uint32 {
return r.entryCount
}
func (r *Reader) ClusterCount() uint32 {
return r.clusterCount
}
func (r *Reader) UUID() string {
return r.uuid
}
func (r *Reader) Close() error {
if err := r.rangeReader.Close(); err != nil {
return errors.WithStack(err)
}
return nil
}
func (r *Reader) MainPage() (Entry, error) {
if r.mainPage == 0xffffffff {
return nil, errors.WithStack(ErrNotFound)
}
entry, err := r.EntryAt(int(r.mainPage))
if err != nil {
return nil, errors.WithStack(ErrNotFound)
}
return entry, nil
}
func (r *Reader) Entries() *EntryIterator {
return &EntryIterator{
reader: r,
}
}
func (r *Reader) EntryAt(idx int) (Entry, error) {
if idx >= len(r.urlIndex) || idx < 0 {
return nil, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", idx)
}
entryPtr := r.urlIndex[idx]
entry, err := r.parseEntryAt(int64(entryPtr))
if err != nil {
return nil, errors.WithStack(err)
}
r.cacheEntry(entryPtr, entry)
return entry, nil
}
func (r *Reader) EntryWithFullURL(url string) (Entry, error) {
urlNum, exists := r.urls[url]
if !exists {
return nil, errors.WithStack(ErrNotFound)
}
entry, err := r.EntryAt(urlNum)
if err != nil {
return nil, errors.WithStack(err)
}
return entry, nil
}
func (r *Reader) EntryWithURL(ns Namespace, url string) (Entry, error) {
fullURL := toFullURL(ns, url)
entry, err := r.EntryWithFullURL(fullURL)
if err != nil {
return nil, errors.WithStack(err)
}
return entry, nil
}
func (r *Reader) EntryWithTitle(ns Namespace, title string) (Entry, error) {
entry, found := r.getEntryByTitleFromCache(ns, title)
if found {
logger.Debug(context.Background(), "found entry with title from cache", logger.F("entry", entry.FullURL()))
return entry, nil
}
iterator := r.Entries()
for iterator.Next() {
entry := iterator.Entry()
if entry.Title() == title && entry.Namespace() == ns {
return entry, nil
}
}
if err := iterator.Err(); err != nil {
return nil, errors.WithStack(err)
}
return nil, errors.WithStack(ErrNotFound)
}
func (r *Reader) getURLCacheKey(fullURL string) string {
return "url:" + fullURL
}
func (r *Reader) getTitleCacheKey(ns Namespace, title string) string {
return fmt.Sprintf("title:%s/%s", ns, title)
}
func (r *Reader) cacheEntry(offset uint64, entry Entry) {
urlKey := r.getURLCacheKey(entry.FullURL())
titleKey := r.getTitleCacheKey(entry.Namespace(), entry.Title())
_, urlFound := r.cache.Peek(urlKey)
_, titleFound := r.cache.Peek(titleKey)
if urlFound && titleFound {
return
}
r.cache.Add(urlKey, entry)
r.cache.Add(titleKey, entry)
}
func (r *Reader) getEntryByTitleFromCache(namespace Namespace, title string) (Entry, bool) {
key := r.getTitleCacheKey(namespace, title)
return r.cache.Get(key)
}
func (r *Reader) parse() error {
if err := r.parseHeader(); err != nil {
return errors.WithStack(err)
}
if err := r.parseMimeTypes(); err != nil {
return errors.WithStack(err)
}
if err := r.parseURLIndex(); err != nil {
return errors.WithStack(err)
}
if err := r.parseClusterIndex(); err != nil {
return errors.WithStack(err)
}
return nil
}
func (r *Reader) parseHeader() error {
header := make([]byte, 80)
if err := r.readRange(0, header); err != nil {
return errors.WithStack(err)
}
magicNumber, err := readUint32(header[0:4], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
if magicNumber != zimFormatMagicNumber {
return errors.Errorf("invalid zim magic number '%d'", magicNumber)
}
majorVersion, err := readUint16(header[4:6], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.majorVersion = majorVersion
minorVersion, err := readUint16(header[6:8], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.minorVersion = minorVersion
if err := r.parseUUID(header[8:16]); err != nil {
return errors.WithStack(err)
}
entryCount, err := readUint32(header[24:28], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.entryCount = entryCount
clusterCount, err := readUint32(header[28:32], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.clusterCount = clusterCount
urlPtrPos, err := readUint64(header[32:40], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.urlPtrPos = urlPtrPos
titlePtrPos, err := readUint64(header[40:48], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.titlePtrPos = titlePtrPos
clusterPtrPos, err := readUint64(header[48:56], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.clusterPtrPos = clusterPtrPos
mimeListPos, err := readUint64(header[56:64], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.mimeListPos = mimeListPos
mainPage, err := readUint32(header[64:68], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.mainPage = mainPage
layoutPage, err := readUint32(header[68:72], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.layoutPage = layoutPage
checksumPos, err := readUint64(header[72:80], binary.LittleEndian)
if err != nil {
return errors.WithStack(err)
}
r.checksumPos = checksumPos
return nil
}
func (r *Reader) parseUUID(data []byte) error {
parts := make([]string, 0, 5)
val32, err := readUint32(data[0:4], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
parts = append(parts, fmt.Sprintf("%08x", val32))
val16, err := readUint16(data[4:6], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
parts = append(parts, fmt.Sprintf("%04x", val16))
val16, err = readUint16(data[6:8], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
parts = append(parts, fmt.Sprintf("%04x", val16))
val16, err = readUint16(data[8:10], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
parts = append(parts, fmt.Sprintf("%04x", val16))
val32, err = readUint32(data[10:14], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
val16, err = readUint16(data[14:16], binary.BigEndian)
if err != nil {
return errors.WithStack(err)
}
parts = append(parts, fmt.Sprintf("%x%x", val32, val16))
r.uuid = strings.Join(parts, "-")
return nil
}
func (r *Reader) parseMimeTypes() error {
mimeTypes := make([]string, 0)
offset := int64(r.mimeListPos)
read := int64(0)
var err error
var found []string
for {
found, read, err = r.readStringsAt(offset+read, 64, 1024)
if err != nil && !errors.Is(err, io.EOF) {
return errors.WithStack(err)
}
if len(found) == 0 || found[0] == "" {
break
}
mimeTypes = append(mimeTypes, found...)
}
r.mimeTypes = mimeTypes
return nil
}
func (r *Reader) parseURLIndex() error {
urlIndex, err := r.parsePointerIndex(int64(r.urlPtrPos), int64(r.entryCount))
if err != nil {
return errors.WithStack(err)
}
r.urlIndex = urlIndex
return nil
}
func (r *Reader) parseClusterIndex() error {
clusterIndex, err := r.parsePointerIndex(int64(r.clusterPtrPos), int64(r.clusterCount+1))
if err != nil {
return errors.WithStack(err)
}
r.clusterIndex = clusterIndex
return nil
}
func (r *Reader) parseEntryAt(offset int64) (Entry, error) {
base, err := r.parseBaseEntry(offset)
if err != nil {
return nil, errors.WithStack(err)
}
var entry Entry
if base.mimeTypeIndex == zimRedirect {
entry, err = r.parseRedirectEntry(offset, base)
if err != nil {
return nil, errors.WithStack(err)
}
} else {
entry, err = r.parseContentEntry(offset, base)
if err != nil {
return nil, errors.WithStack(err)
}
}
return entry, nil
}
func (r *Reader) parsePointerIndex(startAddr int64, count int64) ([]uint64, error) {
index := make([]uint64, count)
data := make([]byte, count*8)
if err := r.readRange(startAddr, data); err != nil {
return nil, errors.WithStack(err)
}
for i := int64(0); i < count; i++ {
offset := i * 8
ptr, err := readUint64(data[offset:offset+8], binary.LittleEndian)
if err != nil {
return nil, errors.WithStack(err)
}
index[i] = ptr
}
return index, nil
}
func (r *Reader) getClusterOffsets(clusterNum int) (uint64, uint64, error) {
if clusterNum > len(r.clusterIndex)-1 || clusterNum < 0 {
return 0, 0, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", clusterNum)
}
return r.clusterIndex[clusterNum], r.clusterIndex[clusterNum+1] - 1, nil
}
func (r *Reader) preload() error {
r.urls = make(map[string]int, r.entryCount)
iterator := r.Entries()
for iterator.Next() {
entry := iterator.Entry()
r.urls[entry.FullURL()] = iterator.Index()
}
if err := iterator.Err(); err != nil {
return errors.WithStack(err)
}
return nil
}
func (r *Reader) readRange(offset int64, v []byte) error {
read, err := r.rangeReader.ReadAt(v, offset)
if err != nil {
return errors.WithStack(err)
}
if read != len(v) {
return io.EOF
}
return nil
}
func (r *Reader) readStringsAt(offset int64, count int, bufferSize int) ([]string, int64, error) {
var sb strings.Builder
read := int64(0)
values := make([]string, 0, count)
wasNullByte := false
for {
data := make([]byte, bufferSize)
err := r.readRange(offset+read, data)
if err != nil && !errors.Is(err, io.EOF) {
return nil, read, errors.WithStack(err)
}
for idx := 0; idx < len(data); idx++ {
d := data[idx]
if err := sb.WriteByte(d); err != nil {
return nil, read, errors.WithStack(err)
}
read++
if d == nullByte {
if wasNullByte {
return values, read, nil
}
wasNullByte = true
str := strings.TrimRight(sb.String(), "\x00")
values = append(values, str)
if len(values) == count || errors.Is(err, io.EOF) {
return values, read, nil
}
sb.Reset()
} else {
wasNullByte = false
}
}
}
}
type RangeReadCloser interface {
io.Closer
ReadAt(data []byte, offset int64) (n int, err error)
}
func NewReader(rangeReader RangeReadCloser, funcs ...OptionFunc) (*Reader, error) {
opts := NewOptions(funcs...)
cache, err := lru.New[string, Entry](opts.CacheSize)
if err != nil {
return nil, errors.WithStack(err)
}
reader := &Reader{
rangeReader: rangeReader,
cache: cache,
}
if err := reader.parse(); err != nil {
return nil, errors.WithStack(err)
}
if err := reader.preload(); err != nil {
return nil, errors.WithStack(err)
}
return reader, nil
}
func Open(path string, funcs ...OptionFunc) (*Reader, error) {
file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
if err != nil {
return nil, errors.WithStack(err)
}
reader, err := NewReader(file, funcs...)
if err != nil {
return nil, errors.WithStack(err)
}
return reader, nil
}

View File

@ -0,0 +1,133 @@
package zim
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type readerTestCase struct {
UUID string `json:"uuid"`
EntryCount uint32 `json:"entryCount"`
Entries []struct {
Namespace Namespace `json:"namespace"`
URL string `json:"url"`
Size int64 `json:"size"`
Compression int `json:"compression"`
MimeType string `json:"mimeType"`
Title string `json:"title"`
} `json:"entries"`
}
func TestReader(t *testing.T) {
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
logger.SetFormat(logger.FormatHuman)
}
files, err := filepath.Glob("testdata/*.zim")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
for _, zf := range files {
testName := filepath.Base(zf)
testCase, err := loadZimFileTestCase(zf)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
t.Run(testName, func(t *testing.T) {
reader, err := Open(zf)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer func() {
if err := reader.Close(); err != nil {
t.Errorf("%+v", errors.WithStack(err))
}
}()
if e, g := testCase.UUID, reader.UUID(); e != g {
t.Errorf("reader.UUID(): expected '%s', got '%s'", e, g)
}
if e, g := testCase.EntryCount, reader.EntryCount(); e != g {
t.Errorf("reader.EntryCount(): expected '%v', got '%v'", e, g)
}
if testCase.Entries == nil {
return
}
for _, entryTestCase := range testCase.Entries {
testName := fmt.Sprintf("Entry/%s/%s", entryTestCase.Namespace, entryTestCase.URL)
t.Run(testName, func(t *testing.T) {
entry, err := reader.EntryWithURL(entryTestCase.Namespace, entryTestCase.URL)
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
content, err := entry.Redirect()
if err != nil {
t.Errorf("%+v", errors.WithStack(err))
}
if e, g := entryTestCase.MimeType, content.MimeType(); e != g {
t.Errorf("content.MimeType(): expected '%v', got '%v'", e, g)
}
if e, g := entryTestCase.Title, content.Title(); e != g {
t.Errorf("content.Title(): expected '%v', got '%v'", e, g)
}
compression, err := content.Compression()
if err != nil {
t.Errorf("%+v", errors.WithStack(err))
}
if e, g := entryTestCase.Compression, compression; e != g {
t.Errorf("content.Compression(): expected '%v', got '%v'", e, g)
}
contentReader, err := content.Reader()
if err != nil {
t.Errorf("%+v", errors.WithStack(err))
}
size, err := contentReader.Size()
if err != nil {
t.Errorf("%+v", errors.WithStack(err))
}
if e, g := entryTestCase.Size, size; e != g {
t.Errorf("content.Size(): expected '%v', got '%v'", e, g)
}
})
}
})
}
}
func loadZimFileTestCase(zimFile string) (*readerTestCase, error) {
testCaseFile, _ := strings.CutSuffix(zimFile, ".zim")
data, err := os.ReadFile(testCaseFile + ".json")
if err != nil {
return nil, errors.WithStack(err)
}
testCase := &readerTestCase{}
if err := json.Unmarshal(data, testCase); err != nil {
return nil, errors.WithStack(err)
}
return testCase, nil
}

View File

@ -0,0 +1,14 @@
{
"uuid": "8d141c3b-115d-bf73-294a-ee3c2e6b97b0",
"entryCount": 6223,
"entries": [
{
"namespace": "C",
"url": "users_page=9",
"compression": 5,
"size": 58646,
"mimeType": "text/html",
"title": "users_page=9"
}
]
}

Binary file not shown.

22
pkg/bundle/zim/testdata/cadoles.json vendored Normal file
View File

@ -0,0 +1,22 @@
{
"uuid": "cf81f094-d802-c790-b854-c74ad9701ddb",
"entryCount": 271,
"entries": [
{
"namespace": "C",
"url": "blog/202206-ShowroomInnovation.jpg",
"compression": 1,
"size": 260260,
"mimeType": "image/jpeg",
"title": "blog/202206-ShowroomInnovation.jpg"
},
{
"namespace": "C",
"url": "team/index.html",
"compression": 5,
"size": 93185,
"mimeType": "text/html",
"title": "Cadoles - Notre équipe"
}
]
}

BIN
pkg/bundle/zim/testdata/cadoles.zim vendored Normal file

Binary file not shown.

View File

@ -0,0 +1,14 @@
{
"uuid": "ad4f406c-2021-2db8-c729-297568bbe376",
"entryCount": 330,
"entries": [
{
"namespace": "M",
"url": "Illustration_48x48@1",
"compression": 5,
"size": 5365,
"mimeType": "text/plain",
"title": "Illustration_48x48@1"
}
]
}

Binary file not shown.

View File

@ -0,0 +1,86 @@
package zim
import (
"io"
"sync"
"github.com/pkg/errors"
)
type UncompressedBlobReader struct {
reader *Reader
blobStartOffset uint64
blobEndOffset uint64
blobSize int
readOffset int
blobData []byte
loadBlobOnce sync.Once
loadBlobErr error
}
// Size implements BlobReader.
func (r *UncompressedBlobReader) Size() (int64, error) {
return int64(r.blobEndOffset - r.blobStartOffset), nil
}
// Close implements io.ReadCloser.
func (r *UncompressedBlobReader) Close() error {
clear(r.blobData)
return nil
}
// Read implements io.ReadCloser.
func (r *UncompressedBlobReader) Read(p []byte) (n int, err error) {
blobData, err := r.loadBlob()
if err != nil {
return 0, errors.WithStack(err)
}
chunkLength := len(p)
remaining := int(len(blobData) - r.readOffset)
if chunkLength > remaining {
chunkLength = remaining
}
chunk := blobData[r.readOffset : r.readOffset+chunkLength]
r.readOffset += chunkLength
copy(p, chunk)
if chunkLength == remaining {
return chunkLength, io.EOF
}
return chunkLength, nil
}
func (r *UncompressedBlobReader) loadBlob() ([]byte, error) {
r.loadBlobOnce.Do(func() {
data := make([]byte, r.blobEndOffset-r.blobStartOffset)
err := r.reader.readRange(int64(r.blobStartOffset), data)
if err != nil {
r.loadBlobErr = errors.WithStack(err)
return
}
r.blobData = data
})
if r.loadBlobErr != nil {
return nil, errors.WithStack(r.loadBlobErr)
}
return r.blobData, nil
}
func NewUncompressedBlobReader(reader *Reader, blobStartOffset, blobEndOffset uint64, blobSize int) *UncompressedBlobReader {
return &UncompressedBlobReader{
reader: reader,
blobStartOffset: blobStartOffset,
blobEndOffset: blobEndOffset,
blobSize: blobSize,
readOffset: 0,
}
}
var _ BlobReader = &UncompressedBlobReader{}

52
pkg/bundle/zim/util.go Normal file
View File

@ -0,0 +1,52 @@
package zim
import (
"bytes"
"encoding/binary"
"github.com/pkg/errors"
)
// read a little endian uint64
func readUint64(b []byte, order binary.ByteOrder) (uint64, error) {
var v uint64
buf := bytes.NewBuffer(b)
if err := binary.Read(buf, order, &v); err != nil {
return 0, errors.WithStack(err)
}
return v, nil
}
// read a little endian uint32
func readUint32(b []byte, order binary.ByteOrder) (uint32, error) {
var v uint32
buf := bytes.NewBuffer(b)
if err := binary.Read(buf, order, &v); err != nil {
return 0, errors.WithStack(err)
}
return v, nil
}
// read a little endian uint16
func readUint16(b []byte, order binary.ByteOrder) (uint16, error) {
var v uint16
buf := bytes.NewBuffer(b)
if err := binary.Read(buf, order, &v); err != nil {
return 0, errors.WithStack(err)
}
return v, nil
}
// read a little endian uint8
func readUint8(b []byte, order binary.ByteOrder) (uint8, error) {
var v uint8
buf := bytes.NewBuffer(b)
if err := binary.Read(buf, order, &v); err != nil {
return 0, errors.WithStack(err)
}
return v, nil
}

View File

@ -0,0 +1,42 @@
package zim
import (
"io"
"github.com/pkg/errors"
"github.com/ulikunitz/xz"
)
type XZBlobReader struct {
decoder *xz.Reader
}
// Close implements io.ReadCloser.
func (r *XZBlobReader) Close() error {
return nil
}
// Read implements io.ReadCloser.
func (r *XZBlobReader) Read(p []byte) (n int, err error) {
return r.decoder.Read(p)
}
var _ io.ReadCloser = &XZBlobReader{}
func NewXZBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
return NewCompressedBlobReader(
reader,
func(r io.Reader) (io.ReadCloser, error) {
decoder, err := xz.NewReader(r)
if err != nil {
return nil, errors.WithStack(err)
}
return &XZBlobReader{decoder}, nil
},
clusterStartOffset,
clusterEndOffset,
blobIndex,
blobSize,
)
}

View File

@ -0,0 +1,43 @@
package zim
import (
"io"
"github.com/klauspost/compress/zstd"
"github.com/pkg/errors"
)
type ZstdBlobReader struct {
decoder *zstd.Decoder
}
// Close implements io.ReadCloser.
func (r *ZstdBlobReader) Close() error {
r.decoder.Close()
return nil
}
// Read implements io.ReadCloser.
func (r *ZstdBlobReader) Read(p []byte) (n int, err error) {
return r.decoder.Read(p)
}
var _ io.ReadCloser = &ZstdBlobReader{}
func NewZStdBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
return NewCompressedBlobReader(
reader,
func(r io.Reader) (io.ReadCloser, error) {
decoder, err := zstd.NewReader(r)
if err != nil {
return nil, errors.WithStack(err)
}
return &ZstdBlobReader{decoder}, nil
},
clusterStartOffset,
clusterEndOffset,
blobIndex,
blobSize,
)
}

483
pkg/bundle/zim_bundle.go Normal file
View File

@ -0,0 +1,483 @@
package bundle
import (
"bytes"
"context"
"io"
"io/fs"
"os"
"path/filepath"
"strings"
"sync"
"time"
"golang.org/x/net/html"
"forge.cadoles.com/arcad/edge/pkg/bundle/zim"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"gopkg.in/yaml.v2"
)
type ZimBundle struct {
archivePath string
initOnce sync.Once
initErr error
reader *zim.Reader
urlNamespaceCache *lru.Cache[string, zim.Namespace]
}
func (b *ZimBundle) File(filename string) (io.ReadCloser, os.FileInfo, error) {
ctx := logger.With(
context.Background(),
logger.F("filename", filename),
)
logger.Debug(ctx, "opening file")
switch filename {
case "manifest.yml":
return b.renderFakeManifest(ctx)
case "server/main.js":
return b.renderFakeServerMain(ctx)
case "public":
return b.renderDirectory(ctx, filename)
case "public/index.html":
return b.renderMainPage(ctx, filename)
default:
return b.renderURL(ctx, filename)
}
}
func (b *ZimBundle) Dir(dirname string) ([]os.FileInfo, error) {
files := make([]os.FileInfo, 0)
return files, nil
}
func (b *ZimBundle) renderFakeManifest(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
if err := b.init(); err != nil {
return nil, nil, errors.WithStack(err)
}
metadata, err := b.reader.Metadata()
if err != nil {
return nil, nil, errors.WithStack(err)
}
manifest := map[string]any{}
manifest["version"] = "0.0.0"
if name, exists := metadata[zim.MetadataName]; exists {
replacer := strings.NewReplacer(
"_", "",
" ", "",
)
manifest["id"] = strings.ToLower(replacer.Replace(name)) + ".zim.edge.app"
} else {
manifest["id"] = b.reader.UUID() + ".zim.edge.app"
}
if title, exists := metadata[zim.MetadataTitle]; exists {
manifest["title"] = title
} else {
manifest["title"] = "Unknown"
}
if description, exists := metadata[zim.MetadataDescription]; exists {
manifest["description"] = description
}
favicon, err := b.reader.Favicon()
if err != nil && !errors.Is(err, zim.ErrNotFound) {
return nil, nil, errors.WithStack(err)
}
if favicon != nil {
manifestMeta, exists := manifest["metadata"].(map[string]any)
if !exists {
manifestMeta = make(map[string]any)
manifest["metadata"] = manifestMeta
}
paths, exists := manifestMeta["paths"].(map[string]any)
if !exists {
paths = make(map[string]any)
manifestMeta["paths"] = paths
}
paths["icon"] = "/" + favicon.FullURL()
}
data, err := yaml.Marshal(manifest)
if err != nil {
return nil, nil, errors.WithStack(err)
}
stat := &zimFileInfo{
isDir: false,
modTime: time.Time{},
mode: 0,
name: "manifest.yml",
size: int64(len(data)),
}
buf := bytes.NewBuffer(data)
file := io.NopCloser(buf)
return file, stat, nil
}
func (b *ZimBundle) renderFakeServerMain(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
stat := &zimFileInfo{
isDir: false,
modTime: time.Time{},
mode: 0,
name: "server/main.js",
size: 0,
}
buf := bytes.NewBuffer(nil)
file := io.NopCloser(buf)
return file, stat, nil
}
func (b *ZimBundle) renderURL(ctx context.Context, url string) (io.ReadCloser, os.FileInfo, error) {
if err := b.init(); err != nil {
return nil, nil, errors.WithStack(err)
}
url = strings.TrimPrefix(url, "public/")
entry, err := b.searchEntryFromURL(ctx, url)
if err != nil {
if errors.Is(err, zim.ErrNotFound) {
return nil, nil, os.ErrNotExist
}
return nil, nil, errors.WithStack(err)
}
logger.Debug(
ctx, "found zim entry",
logger.F("webURL", url),
logger.F("zimFullURL", entry.FullURL()),
)
content, err := entry.Redirect()
if err != nil {
return nil, nil, errors.WithStack(err)
}
contentReader, err := content.Reader()
if err != nil {
return nil, nil, errors.WithStack(err)
}
size, err := contentReader.Size()
if err != nil {
return nil, nil, errors.WithStack(err)
}
filename := filepath.Base(url)
mimeType := content.MimeType()
if mimeType != "text/html" {
zimFile := &zimFile{
fileInfo: &zimFileInfo{
isDir: false,
modTime: time.Time{},
mode: 0,
name: filename,
size: size,
},
reader: contentReader,
}
return zimFile, zimFile.fileInfo, nil
}
// Read HTML file and inject Edge scripts
data, err := io.ReadAll(contentReader)
if err != nil {
return nil, nil, err
}
injected, err := b.injectEdgeScriptTag(data)
if err != nil {
logger.Error(ctx, "could not inject edge script", logger.E(errors.WithStack(err)))
} else {
data = injected
}
zimFile := &zimFile{
fileInfo: &zimFileInfo{
isDir: false,
modTime: time.Time{},
mode: 0,
name: filename,
size: size,
},
reader: io.NopCloser(bytes.NewBuffer(data)),
}
return zimFile, zimFile.fileInfo, nil
}
func (b *ZimBundle) searchEntryFromURL(ctx context.Context, url string) (zim.Entry, error) {
ctx = logger.With(ctx, logger.F("webURL", url))
logger.Debug(ctx, "searching entry namespace in local cache")
entry, err := b.reader.EntryWithFullURL(url)
if err != nil && !errors.Is(err, zim.ErrNotFound) {
return nil, errors.WithStack(err)
}
if entry != nil {
return entry, nil
}
contentNamespaces := []zim.Namespace{
zim.V6NamespaceContent,
zim.V6NamespaceMetadata,
zim.V5NamespaceLayout,
zim.V5NamespaceArticle,
zim.V5NamespaceImageFile,
zim.V5NamespaceMetadata,
}
logger.Debug(
ctx, "make educated guesses about potential url namespace",
logger.F("zimNamespaces", contentNamespaces),
)
for _, ns := range contentNamespaces {
logger.Debug(
ctx, "trying to access entry directly",
logger.F("zimNamespace", ns),
logger.F("zimURL", url),
)
entry, err := b.reader.EntryWithURL(ns, url)
if err != nil && !errors.Is(err, zim.ErrNotFound) {
return nil, errors.WithStack(err)
}
if entry != nil {
b.urlNamespaceCache.Add(url, entry.Namespace())
return entry, nil
}
}
logger.Debug(ctx, "doing full entries scan")
iterator := b.reader.Entries()
for iterator.Next() {
current := iterator.Entry()
if current.FullURL() != url && current.URL() != url {
continue
}
entry = current
b.urlNamespaceCache.Add(url, entry.Namespace())
break
}
if err := iterator.Err(); err != nil {
return nil, errors.WithStack(err)
}
if entry == nil {
return nil, errors.WithStack(zim.ErrNotFound)
}
return entry, nil
}
func (b *ZimBundle) renderDirectory(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
zimFile := &zimFile{
fileInfo: &zimFileInfo{
isDir: true,
modTime: time.Time{},
mode: 0,
name: filename,
size: 0,
},
reader: io.NopCloser(bytes.NewBuffer(nil)),
}
return zimFile, zimFile.fileInfo, nil
}
func (b *ZimBundle) renderMainPage(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
if err := b.init(); err != nil {
return nil, nil, errors.WithStack(err)
}
main, err := b.reader.MainPage()
if err != nil {
return nil, nil, errors.WithStack(err)
}
return b.renderURL(ctx, main.FullURL())
}
func (b *ZimBundle) injectEdgeScriptTag(data []byte) ([]byte, error) {
buff := bytes.NewBuffer(data)
doc, err := html.Parse(buff)
if err != nil {
return nil, errors.WithStack(err)
}
var f func(*html.Node) bool
f = func(n *html.Node) bool {
if n.Type == html.ElementNode && n.Data == "head" {
script := &html.Node{
Type: html.ElementNode,
Data: "script",
Attr: []html.Attribute{
{
Key: "src",
Val: "/edge/sdk/client.js",
},
},
}
n.AppendChild(script)
return false
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
if keepWalking := f(c); !keepWalking {
return false
}
}
return true
}
f(doc)
buff.Reset()
if err := html.Render(buff, doc); err != nil {
return nil, errors.WithStack(err)
}
return buff.Bytes(), nil
}
func (b *ZimBundle) init() error {
b.initOnce.Do(func() {
reader, err := zim.Open(b.archivePath)
if err != nil {
b.initErr = errors.Wrapf(err, "could not open '%v'", b.archivePath)
return
}
b.reader = reader
cache, err := lru.New[string, zim.Namespace](128)
if err != nil {
b.initErr = errors.Wrap(err, "could not initialize cache")
return
}
b.urlNamespaceCache = cache
})
if b.initErr != nil {
return errors.WithStack(b.initErr)
}
return nil
}
func NewZimBundle(archivePath string) *ZimBundle {
return &ZimBundle{
archivePath: archivePath,
}
}
type zimFile struct {
fileInfo *zimFileInfo
reader io.ReadCloser
}
// Close implements fs.File.
func (f *zimFile) Close() error {
if err := f.reader.Close(); err != nil {
return errors.WithStack(err)
}
return nil
}
// Read implements fs.File.
func (f *zimFile) Read(d []byte) (int, error) {
n, err := f.reader.Read(d)
if err != nil {
if errors.Is(err, io.EOF) {
return n, err
}
return n, errors.WithStack(err)
}
return n, nil
}
// Stat implements fs.File.
func (f *zimFile) Stat() (fs.FileInfo, error) {
return f.fileInfo, nil
}
var _ fs.File = &zimFile{}
type zimFileInfo struct {
isDir bool
modTime time.Time
mode fs.FileMode
name string
size int64
}
// IsDir implements fs.FileInfo.
func (i *zimFileInfo) IsDir() bool {
return i.isDir
}
// ModTime implements fs.FileInfo.
func (i *zimFileInfo) ModTime() time.Time {
return i.modTime
}
// Mode implements fs.FileInfo.
func (i *zimFileInfo) Mode() fs.FileMode {
return i.mode
}
// Name implements fs.FileInfo.
func (i *zimFileInfo) Name() string {
return i.name
}
// Size implements fs.FileInfo.
func (i *zimFileInfo) Size() int64 {
return i.size
}
// Sys implements fs.FileInfo.
func (*zimFileInfo) Sys() any {
return nil
}
var _ fs.FileInfo = &zimFileInfo{}

View File

@ -83,8 +83,12 @@ func (d *eventDispatcher) Close() {
}
func (d *eventDispatcher) close() {
d.closed = true
if d.closed {
return
}
close(d.in)
d.closed = true
}
func (d *eventDispatcher) In(msg bus.Message) (err error) {
@ -149,7 +153,7 @@ func (d *eventDispatcher) Run(ctx context.Context) {
ctx,
"message subscription context canceled",
logger.F("message", msg),
logger.E(errors.WithStack(ctx.Err())),
logger.CapturedE(errors.WithStack(ctx.Err())),
)
return

View File

@ -39,7 +39,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, h.uploadMaxFileSize)
if err := r.ParseMultipartForm(h.uploadMaxFileSize); err != nil {
logger.Error(ctx, "could not parse multipart form", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not parse multipart form", logger.CapturedE(errors.WithStack(err)))
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
return
@ -47,7 +47,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
_, fileHeader, err := r.FormFile("file")
if err != nil {
logger.Error(ctx, "could not read form file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not read form file", logger.CapturedE(errors.WithStack(err)))
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
return
@ -58,7 +58,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
rawMetadata := r.Form.Get("metadata")
if rawMetadata != "" {
if err := json.Unmarshal([]byte(rawMetadata), &metadata); err != nil {
logger.Error(ctx, "could not parse metadata", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not parse metadata", logger.CapturedE(errors.WithStack(err)))
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
return
@ -73,7 +73,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
logger.Error(ctx, "could not retrieve file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve file", logger.CapturedE(errors.WithStack(err)))
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
@ -125,7 +125,7 @@ func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
logger.Error(ctx, "could not retrieve file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve file", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -135,7 +135,7 @@ func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
if !ok {
logger.Error(
ctx, "unexpected download response message",
logger.E(errors.WithStack(bus.ErrUnexpectedMessage)),
logger.CapturedE(errors.WithStack(bus.ErrUnexpectedMessage)),
logger.F("message", reply),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
@ -157,7 +157,7 @@ func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := replyMsg.Blob.Close(); err != nil {
logger.Error(ctx, "could not close blob", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close blob", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -175,7 +175,7 @@ func serveFile(w http.ResponseWriter, r *http.Request, fs fs.FS, path string) {
return
}
logger.Error(ctx, "error while opening fs file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error while opening fs file", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -183,13 +183,13 @@ func serveFile(w http.ResponseWriter, r *http.Request, fs fs.FS, path string) {
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "error while closing fs file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error while closing fs file", logger.CapturedE(errors.WithStack(err)))
}
}()
info, err := file.Stat()
if err != nil {
logger.Error(ctx, "error while retrieving fs file stat", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error while retrieving fs file stat", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return

View File

@ -34,7 +34,7 @@ func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
logger.Error(ctx, "could not retrieve fetch request reply", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve fetch request reply", logger.CapturedE(errors.WithStack(err)))
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
@ -63,7 +63,7 @@ func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
if err != nil {
logger.Error(
ctx, "could not create proxy request",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
@ -82,7 +82,7 @@ func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
if err != nil {
logger.Error(
ctx, "could not execute proxy request",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
@ -93,7 +93,7 @@ func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
if err := res.Body.Close(); err != nil {
logger.Error(
ctx, "could not close response body",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
}
}()

View File

@ -1,7 +1,8 @@
package http
import (
"io/ioutil"
"context"
"io"
"net/http"
"sync"
@ -40,7 +41,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
h.router.ServeHTTP(w, r)
}
func (h *Handler) Load(bdle bundle.Bundle) error {
func (h *Handler) Load(ctx context.Context, bdle bundle.Bundle) error {
h.mutex.Lock()
defer h.mutex.Unlock()
@ -49,7 +50,7 @@ func (h *Handler) Load(bdle bundle.Bundle) error {
return errors.Wrap(err, "could not open server main script")
}
mainScript, err := ioutil.ReadAll(file)
mainScript, err := io.ReadAll(file)
if err != nil {
return errors.Wrap(err, "could not read server main script")
}
@ -68,7 +69,7 @@ func (h *Handler) Load(bdle bundle.Bundle) error {
h.server.Stop()
}
if err := server.Start(); err != nil {
if err := server.Start(ctx); err != nil {
return errors.WithStack(err)
}

View File

@ -27,11 +27,10 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler {
r.URL.Path = "/"
handler.ServeHTTP(w, r)
return
}
logger.Error(r.Context(), "could not open bundle file", logger.E(err))
logger.Error(r.Context(), "could not open bundle file", logger.CapturedE(err))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -39,7 +38,7 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler {
defer func() {
if err := file.Close(); err != nil {
logger.Error(r.Context(), "could not close file", logger.E(err))
logger.Error(r.Context(), "could not close file", logger.CapturedE(err))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)

View File

@ -37,7 +37,7 @@ func (h *Handler) handleSockJSSession(sess sockjs.Session) {
defer func() {
if sess.GetSessionState() == sockjs.SessionActive {
if err := sess.Close(statusChannelClosed, "channel closed"); err != nil {
logger.Error(ctx, "could not close sockjs session", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close sockjs session", logger.CapturedE(errors.WithStack(err)))
}
}
}()
@ -63,7 +63,7 @@ func (h *Handler) handleServerMessages(ctx context.Context, sess sockjs.Session)
}
if err := sess.Close(statusChannelClosed, "channel closed"); err != nil {
logger.Error(ctx, "could not close sockjs session", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close sockjs session", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -96,7 +96,7 @@ func (h *Handler) handleServerMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not encode message",
logger.E(err),
logger.CapturedE(errors.WithStack(err)),
)
continue
@ -112,7 +112,7 @@ func (h *Handler) handleServerMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not encode message",
logger.E(err),
logger.CapturedE(errors.WithStack(err)),
)
continue
@ -125,7 +125,7 @@ func (h *Handler) handleServerMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not send message",
logger.E(err),
logger.CapturedE(errors.WithStack(err)),
)
}
}
@ -152,7 +152,7 @@ func (h *Handler) handleClientMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not read message",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
break
@ -165,7 +165,7 @@ func (h *Handler) handleClientMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not decode message",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
break
@ -179,7 +179,7 @@ func (h *Handler) handleClientMessages(ctx context.Context, sess sockjs.Session)
logger.Error(
ctx,
"could not decode payload",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
)
return
@ -197,7 +197,7 @@ func (h *Handler) handleClientMessages(ctx context.Context, sess sockjs.Session)
if err := h.bus.Publish(ctx, clientMessage); err != nil {
logger.Error(ctx, "could not publish message",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("message", clientMessage),
)

View File

@ -1,5 +1,6 @@
package auth
package jwtutil
import "errors"
var ErrUnauthenticated = errors.New("unauthenticated")
var ErrNoKeySet = errors.New("no keyset")

71
pkg/jwtutil/io.go Normal file
View File

@ -0,0 +1,71 @@
package jwtutil
import (
"crypto/rand"
"crypto/rsa"
"os"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/pkg/errors"
)
func LoadOrGenerateKey(path string, defaultKeySize int) (jwk.Key, error) {
key, err := LoadKey(path)
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
key, err = GenerateKey(defaultKeySize)
if err != nil {
return nil, errors.WithStack(err)
}
if err := SaveKey(path, key); err != nil {
return nil, errors.WithStack(err)
}
}
return key, nil
}
func LoadKey(path string) (jwk.Key, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, errors.WithStack(err)
}
key, err := jwk.ParseKey(data, jwk.WithPEM(true))
if err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}
func SaveKey(path string, key jwk.Key) error {
data, err := jwk.Pem(key)
if err != nil {
return errors.WithStack(err)
}
if err := os.WriteFile(path, data, os.FileMode(0600)); err != nil {
return errors.WithStack(err)
}
return nil
}
func GenerateKey(keySize int) (jwk.Key, error) {
rsaKey, err := rsa.GenerateKey(rand.Reader, keySize)
if err != nil {
return nil, errors.WithStack(err)
}
key, err := jwk.FromRaw(rsaKey)
if err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}

77
pkg/jwtutil/key.go Normal file
View File

@ -0,0 +1,77 @@
package jwtutil
import (
"strings"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/pkg/errors"
)
func AddKeyWithSigningAlgo(keySet jwk.Set, key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) error {
addedKey := key
if !strings.HasPrefix(string(signingAlgorithm), "HS") {
publicKey, err := key.PublicKey()
if err != nil {
return errors.WithStack(err)
}
addedKey = publicKey
}
if err := addedKey.Set(jwk.AlgorithmKey, signingAlgorithm); err != nil {
return errors.WithStack(err)
}
if err := keySet.AddKey(addedKey); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewKeySet(keys ...jwk.Key) (jwk.Set, error) {
set := jwk.NewSet()
for _, k := range keys {
if err := set.AddKey(k); err != nil {
return nil, errors.WithStack(err)
}
}
return set, nil
}
func NewSymmetricKey(secret []byte) (jwk.Key, error) {
key, err := jwk.FromRaw(secret)
if err != nil {
return nil, errors.WithStack(err)
}
if err := key.Set(jwk.AlgorithmKey, jwa.HS256); err != nil {
return nil, errors.WithStack(err)
}
return key, nil
}
func NewSymmetricKeySet(secrets ...[]byte) (jwk.Set, error) {
keys := make([]jwk.Key, len(secrets))
for idx, sec := range secrets {
key, err := NewSymmetricKey(sec)
if err != nil {
return nil, errors.WithStack(err)
}
keys[idx] = key
}
keySet, err := NewKeySet(keys...)
if err != nil {
return nil, errors.WithStack(err)
}
return keySet, nil
}

119
pkg/jwtutil/request.go Normal file
View File

@ -0,0 +1,119 @@
package jwtutil
import (
"net/http"
"strings"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/lestrrat-go/jwx/v2/jwt"
"github.com/pkg/errors"
)
type TokenFinderFunc func(r *http.Request) (string, error)
type FindTokenOptions struct {
Finders []TokenFinderFunc
}
type FindTokenOptionFunc func(*FindTokenOptions)
type GetKeySetFunc func() (jwk.Set, error)
func WithFinders(finders ...TokenFinderFunc) FindTokenOptionFunc {
return func(opts *FindTokenOptions) {
opts.Finders = finders
}
}
func NewFindTokenOptions(funcs ...FindTokenOptionFunc) *FindTokenOptions {
opts := &FindTokenOptions{
Finders: []TokenFinderFunc{
FindTokenFromAuthorizationHeader,
},
}
for _, fn := range funcs {
fn(opts)
}
return opts
}
func FindTokenFromAuthorizationHeader(r *http.Request) (string, error) {
authorization := r.Header.Get("Authorization")
// Retrieve token from Authorization header
rawToken := strings.TrimPrefix(authorization, "Bearer ")
return rawToken, nil
}
func FindTokenFromQueryString(name string) TokenFinderFunc {
return func(r *http.Request) (string, error) {
return r.URL.Query().Get(name), nil
}
}
func FindTokenFromCookie(cookieName string) TokenFinderFunc {
return func(r *http.Request) (string, error) {
cookie, err := r.Cookie(cookieName)
if err != nil && !errors.Is(err, http.ErrNoCookie) {
return "", errors.WithStack(err)
}
if cookie == nil {
return "", nil
}
return cookie.Value, nil
}
}
func FindRawToken(r *http.Request, funcs ...FindTokenOptionFunc) (string, error) {
opts := NewFindTokenOptions(funcs...)
var rawToken string
var err error
for _, find := range opts.Finders {
rawToken, err = find(r)
if err != nil {
return "", errors.WithStack(err)
}
if rawToken == "" {
continue
}
break
}
if rawToken == "" {
return "", errors.WithStack(ErrUnauthenticated)
}
return rawToken, nil
}
func FindToken(r *http.Request, getKeySet GetKeySetFunc, funcs ...FindTokenOptionFunc) (jwt.Token, error) {
rawToken, err := FindRawToken(r, funcs...)
if err != nil {
return nil, errors.WithStack(err)
}
keySet, err := getKeySet()
if err != nil {
return nil, errors.WithStack(err)
}
if keySet == nil {
return nil, errors.WithStack(ErrNoKeySet)
}
token, err := Parse([]byte(rawToken), keySet)
if err != nil {
return nil, errors.WithStack(err)
}
return token, nil
}

53
pkg/jwtutil/token.go Normal file
View File

@ -0,0 +1,53 @@
package jwtutil
import (
"time"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/lestrrat-go/jwx/v2/jws"
"github.com/lestrrat-go/jwx/v2/jwt"
"github.com/oklog/ulid/v2"
"github.com/pkg/errors"
)
func SignedToken(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, claims map[string]any) ([]byte, error) {
token := jwt.New()
if err := token.Set(jwt.NotBeforeKey, time.Now()); err != nil {
return nil, errors.WithStack(err)
}
if err := token.Set(jwt.JwtIDKey, ulid.Make().String()); err != nil {
return nil, errors.WithStack(err)
}
for key, value := range claims {
if err := token.Set(key, value); err != nil {
return nil, errors.Wrapf(err, "could not set claim '%s' with value '%v'", key, value)
}
}
if err := token.Set(jwk.AlgorithmKey, signingAlgorithm); err != nil {
return nil, errors.WithStack(err)
}
rawToken, err := jwt.Sign(token, jwt.WithKey(signingAlgorithm, key))
if err != nil {
return nil, errors.WithStack(err)
}
return rawToken, nil
}
func Parse(rawToken []byte, keySet jwk.Set) (jwt.Token, error) {
token, err := jwt.Parse(rawToken,
jwt.WithKeySet(keySet, jws.WithRequireKid(false)),
jwt.WithValidate(true),
)
if err != nil {
return nil, errors.WithStack(err)
}
return token, nil
}

View File

@ -3,7 +3,7 @@ package memory
import (
"context"
"fmt"
"io/ioutil"
"os"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
@ -41,7 +41,7 @@ func TestAppModuleWithMemoryRepository(t *testing.T) {
file := "testdata/app.js"
data, err := ioutil.ReadFile(file)
data, err := os.ReadFile(file)
if err != nil {
t.Fatal(err)
}
@ -52,7 +52,8 @@ func TestAppModuleWithMemoryRepository(t *testing.T) {
defer server.Stop()
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
}

View File

@ -20,7 +20,7 @@ type Handler struct {
func (h *Handler) serveApps(w http.ResponseWriter, r *http.Request) {
manifests, err := h.repo.List(r.Context())
if err != nil {
logger.Error(r.Context(), "could not retrieve app manifest", logger.E(errors.WithStack(err)))
logger.Error(r.Context(), "could not retrieve app manifest", logger.CapturedE(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
@ -44,7 +44,7 @@ func (h *Handler) serveApp(w http.ResponseWriter, r *http.Request) {
return
}
logger.Error(r.Context(), "could not retrieve app manifest", logger.E(errors.WithStack(err)))
logger.Error(r.Context(), "could not retrieve app manifest", logger.CapturedE(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return
@ -84,7 +84,7 @@ func (h *Handler) serveAppURL(w http.ResponseWriter, r *http.Request) {
return
}
logger.Error(r.Context(), "could not retrieve app url", logger.E(errors.WithStack(err)))
logger.Error(r.Context(), "could not retrieve app url", logger.CapturedE(errors.WithStack(err)))
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
return

View File

@ -7,9 +7,9 @@ import (
_ "embed"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
"forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd"
"forge.cadoles.com/arcad/edge/pkg/module/auth/jwt"
"github.com/go-chi/chi/v5"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
@ -31,12 +31,12 @@ func init() {
}
type LocalHandler struct {
router chi.Router
algo jwa.KeyAlgorithm
key jwk.Key
getCookieDomain GetCookieDomainFunc
cookieDuration time.Duration
accounts map[string]LocalAccount
router chi.Router
key jwk.Key
signingAlgorithm jwa.SignatureAlgorithm
getCookieDomain GetCookieDomainFunc
cookieDuration time.Duration
accounts map[string]LocalAccount
}
func (h *LocalHandler) initRouter(prefix string) {
@ -69,7 +69,7 @@ func (h *LocalHandler) serveForm(w http.ResponseWriter, r *http.Request) {
}
if err := loginTemplate.Execute(w, data); err != nil {
logger.Error(ctx, "could not execute login page template", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not execute login page template", logger.CapturedE(errors.WithStack(err)))
}
}
@ -77,7 +77,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if err := r.ParseForm(); err != nil {
logger.Error(ctx, "could not parse form", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not parse form", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
return
@ -99,13 +99,13 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
data.Message = "Invalid username or password."
if err := loginTemplate.Execute(w, data); err != nil {
logger.Error(ctx, "could not execute login page template", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not execute login page template", logger.CapturedE(errors.WithStack(err)))
}
return
}
logger.Error(ctx, "could not authenticate account", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not authenticate account", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -113,9 +113,9 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
account.Claims[auth.ClaimIssuer] = "local"
token, err := jwt.GenerateSignedToken(h.algo, h.key, account.Claims)
token, err := jwtutil.SignedToken(h.key, h.signingAlgorithm, account.Claims)
if err != nil {
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not generate signed token", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -123,7 +123,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
cookieDomain, err := h.getCookieDomain(r)
if err != nil {
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -146,7 +146,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
func (h *LocalHandler) handleLogout(w http.ResponseWriter, r *http.Request) {
cookieDomain, err := h.getCookieDomain(r)
if err != nil {
logger.Error(r.Context(), "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
logger.Error(r.Context(), "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -182,18 +182,18 @@ func (h *LocalHandler) authenticate(username, password string) (*LocalAccount, e
return &account, nil
}
func NewLocalHandler(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...LocalHandlerOptionFunc) *LocalHandler {
func NewLocalHandler(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs ...LocalHandlerOptionFunc) *LocalHandler {
opts := defaultLocalHandlerOptions()
for _, fn := range funcs {
fn(opts)
}
handler := &LocalHandler{
algo: algo,
key: key,
accounts: toAccountsMap(opts.Accounts),
getCookieDomain: opts.GetCookieDomain,
cookieDuration: opts.CookieDuration,
key: key,
signingAlgorithm: signingAlgorithm,
accounts: toAccountsMap(opts.Accounts),
getCookieDomain: opts.GetCookieDomain,
cookieDuration: opts.CookieDuration,
}
handler.initRouter(opts.RoutePrefix)

View File

@ -1,118 +0,0 @@
package auth
import (
"context"
"net/http"
"strings"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/lestrrat-go/jwx/v2/jws"
"github.com/lestrrat-go/jwx/v2/jwt"
"github.com/pkg/errors"
)
const (
CookieName string = "edge-auth"
)
type GetKeySetFunc func() (jwk.Set, error)
func WithJWT(getKeySet GetKeySetFunc) OptionFunc {
return func(o *Option) {
o.GetClaims = func(ctx context.Context, r *http.Request, names ...string) ([]string, error) {
claim, err := getClaims[string](r, getKeySet, names...)
if err != nil {
return nil, errors.WithStack(err)
}
return claim, nil
}
}
}
func FindRawToken(r *http.Request) (string, error) {
authorization := r.Header.Get("Authorization")
// Retrieve token from Authorization header
rawToken := strings.TrimPrefix(authorization, "Bearer ")
// Retrieve token from ?edge-auth=<value>
if rawToken == "" {
rawToken = r.URL.Query().Get(CookieName)
}
if rawToken == "" {
cookie, err := r.Cookie(CookieName)
if err != nil && !errors.Is(err, http.ErrNoCookie) {
return "", errors.WithStack(err)
}
if cookie != nil {
rawToken = cookie.Value
}
}
if rawToken == "" {
return "", errors.WithStack(ErrUnauthenticated)
}
return rawToken, nil
}
func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) {
rawToken, err := FindRawToken(r)
if err != nil {
return nil, errors.WithStack(err)
}
keySet, err := getKeySet()
if err != nil {
return nil, errors.WithStack(err)
}
if keySet == nil {
return nil, errors.New("no keyset")
}
token, err := jwt.Parse([]byte(rawToken),
jwt.WithKeySet(keySet, jws.WithRequireKid(false)),
jwt.WithValidate(true),
)
if err != nil {
return nil, errors.WithStack(err)
}
return token, nil
}
func getClaims[T any](r *http.Request, getKeySet GetKeySetFunc, names ...string) ([]T, error) {
token, err := FindToken(r, getKeySet)
if err != nil {
return nil, errors.WithStack(err)
}
ctx := r.Context()
mapClaims, err := token.AsMap(ctx)
if err != nil {
return nil, errors.WithStack(err)
}
claims := make([]T, len(names))
for idx, n := range names {
rawClaim, exists := mapClaims[n]
if !exists {
continue
}
claim, ok := rawClaim.(T)
if !ok {
return nil, errors.Errorf("unexpected claim '%s' to be of type '%T', got '%T'", n, new(T), rawClaim)
}
claims[idx] = claim
}
return claims, nil
}

View File

@ -1,35 +0,0 @@
package jwt
import (
"time"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
"github.com/lestrrat-go/jwx/v2/jwt"
"github.com/pkg/errors"
)
func GenerateSignedToken(algo jwa.KeyAlgorithm, key jwk.Key, claims map[string]any) ([]byte, error) {
token := jwt.New()
if err := token.Set(jwt.NotBeforeKey, time.Now()); err != nil {
return nil, errors.WithStack(err)
}
for key, value := range claims {
if err := token.Set(key, value); err != nil {
return nil, errors.Wrapf(err, "could not set claim '%s' with value '%v'", key, value)
}
}
if err := token.Set(jwk.AlgorithmKey, jwa.HS256); err != nil {
return nil, errors.WithStack(err)
}
rawToken, err := jwt.Sign(token, jwt.WithKey(algo, key))
if err != nil {
return nil, errors.WithStack(err)
}
return rawToken, nil
}

View File

@ -7,8 +7,8 @@ import (
"net/http"
"time"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
"forge.cadoles.com/arcad/edge/pkg/module/auth/jwt"
"github.com/google/uuid"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
@ -18,7 +18,7 @@ import (
const AnonIssuer = "anon"
func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOptionFunc) func(next http.Handler) http.Handler {
func AnonymousUser(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs ...AnonymousUserOptionFunc) func(next http.Handler) http.Handler {
opts := defaultAnonymousUserOptions()
for _, fn := range funcs {
fn(opts)
@ -26,7 +26,11 @@ func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOpt
return func(next http.Handler) http.Handler {
handler := func(w http.ResponseWriter, r *http.Request) {
rawToken, err := auth.FindRawToken(r)
rawToken, err := jwtutil.FindRawToken(r, jwtutil.WithFinders(
jwtutil.FindTokenFromAuthorizationHeader,
jwtutil.FindTokenFromQueryString(auth.CookieName),
jwtutil.FindTokenFromCookie(auth.CookieName),
))
// If request already has a raw token, we do nothing
if rawToken != "" && err == nil {
@ -38,7 +42,7 @@ func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOpt
uuid, err := uuid.NewUUID()
if err != nil {
logger.Error(ctx, "could not generate uuid for anonymous user", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not generate uuid for anonymous user", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -47,7 +51,7 @@ func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOpt
subject := fmt.Sprintf("%s-%s", AnonIssuer, uuid.String())
preferredUsername, err := generateRandomPreferredUsername(8)
if err != nil {
logger.Error(ctx, "could not generate preferred username for anonymous user", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not generate preferred username for anonymous user", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -62,9 +66,9 @@ func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOpt
auth.ClaimEdgeTenant: opts.Tenant,
}
token, err := jwt.GenerateSignedToken(algo, key, claims)
token, err := jwtutil.SignedToken(key, signingAlgorithm, claims)
if err != nil {
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not generate signed token", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
@ -72,7 +76,7 @@ func AnonymousUser(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...AnonymousUserOpt
cookieDomain, err := opts.GetCookieDomain(r)
if err != nil {
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return

View File

@ -5,12 +5,17 @@ import (
"forge.cadoles.com/arcad/edge/pkg/app"
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/module/util"
"github.com/dop251/goja"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
const (
CookieName string = "edge-auth"
)
const (
ClaimSubject = "sub"
ClaimIssuer = "iss"
@ -21,8 +26,8 @@ const (
)
type Module struct {
server *app.Server
getClaims GetClaimsFunc
server *app.Server
getClaimFn GetClaimFunc
}
func (m *Module) Name() string {
@ -68,21 +73,17 @@ func (m *Module) getClaim(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
panic(rt.ToValue(errors.New("could not find http request in context")))
}
claim, err := m.getClaims(ctx, req, claimName)
claim, err := m.getClaimFn(ctx, req, claimName)
if err != nil {
if errors.Is(err, ErrUnauthenticated) {
if errors.Is(err, jwtutil.ErrUnauthenticated) {
return nil
}
logger.Error(ctx, "could not retrieve claim", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not retrieve claim", logger.CapturedE(errors.WithStack(err)))
return nil
}
if len(claim) == 0 || claim[0] == "" {
return nil
}
return rt.ToValue(claim[0])
return rt.ToValue(claim)
}
func ModuleFactory(funcs ...OptionFunc) app.ServerModuleFactory {
@ -93,8 +94,8 @@ func ModuleFactory(funcs ...OptionFunc) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
return &Module{
server: server,
getClaims: opt.GetClaims,
server: server,
getClaimFn: opt.GetClaim,
}
}
}

View File

@ -10,6 +10,7 @@ import (
"cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app"
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"forge.cadoles.com/arcad/edge/pkg/module"
"github.com/lestrrat-go/jwx/v2/jwa"
"github.com/lestrrat-go/jwx/v2/jwk"
@ -41,7 +42,8 @@ func TestAuthModule(t *testing.T) {
t.Fatal(err)
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -69,7 +71,7 @@ func TestAuthModule(t *testing.T) {
req.Header.Add("Authorization", "Bearer "+string(rawToken))
ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
ctx = context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
@ -97,7 +99,8 @@ func TestAuthAnonymousModule(t *testing.T) {
t.Fatal(err)
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -108,7 +111,7 @@ func TestAuthAnonymousModule(t *testing.T) {
t.Fatalf("%+v", errors.WithStack(err))
}
ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
ctx = context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
@ -130,7 +133,7 @@ func getDummyKey() jwk.Key {
return key
}
func getDummyKeySet(key jwk.Key) GetKeySetFunc {
func getDummyKeySet(key jwk.Key) jwtutil.GetKeySetFunc {
return func() (jwk.Set, error) {
set := jwk.NewSet()

View File

@ -3,6 +3,7 @@ package auth
import (
"net/http"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"github.com/go-chi/chi/v5"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/api"
@ -12,39 +13,39 @@ import (
type MountFunc func(r chi.Router)
type Handler struct {
getClaims GetClaimsFunc
getClaim GetClaimFunc
profileClaims []string
}
func (h *Handler) serveProfile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
claims, err := h.getClaims(ctx, r, h.profileClaims...)
if err != nil {
if errors.Is(err, ErrUnauthenticated) {
profile := make(map[string]any)
for _, name := range h.profileClaims {
value, err := h.getClaim(ctx, r, name)
if err != nil {
if errors.Is(err, jwtutil.ErrUnauthenticated) {
api.ErrorResponse(
w, http.StatusUnauthorized,
api.ErrCodeUnauthorized,
nil,
)
return
}
logger.Error(ctx, "could not retrieve claims", logger.CapturedE(errors.WithStack(err)))
api.ErrorResponse(
w, http.StatusUnauthorized,
api.ErrCodeUnauthorized,
w, http.StatusInternalServerError,
api.ErrCodeUnknownError,
nil,
)
return
}
logger.Error(ctx, "could not retrieve claims", logger.E(errors.WithStack(err)))
api.ErrorResponse(
w, http.StatusInternalServerError,
api.ErrCodeUnknownError,
nil,
)
return
}
profile := make(map[string]any)
for idx, cl := range h.profileClaims {
profile[cl] = claims[idx]
profile[name] = value
}
api.DataResponse(w, http.StatusOK, struct {
@ -62,7 +63,7 @@ func Mount(authHandler http.Handler, funcs ...OptionFunc) MountFunc {
handler := &Handler{
profileClaims: opt.ProfileClaims,
getClaims: opt.GetClaims,
getClaim: opt.GetClaim,
}
return func(r chi.Router) {

View File

@ -2,15 +2,17 @@ package auth
import (
"context"
"fmt"
"net/http"
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
"github.com/pkg/errors"
)
type GetClaimsFunc func(ctx context.Context, r *http.Request, claims ...string) ([]string, error)
type GetClaimFunc func(ctx context.Context, r *http.Request, name string) (string, error)
type Option struct {
GetClaims GetClaimsFunc
GetClaim GetClaimFunc
ProfileClaims []string
}
@ -18,7 +20,7 @@ type OptionFunc func(*Option)
func defaultOptions() *Option {
return &Option{
GetClaims: dummyGetClaims,
GetClaim: dummyGetClaim,
ProfileClaims: []string{
ClaimSubject,
ClaimIssuer,
@ -30,13 +32,13 @@ func defaultOptions() *Option {
}
}
func dummyGetClaims(ctx context.Context, r *http.Request, claims ...string) ([]string, error) {
return nil, errors.Errorf("dummy getclaim func cannot retrieve claims '%s'", claims)
func dummyGetClaim(ctx context.Context, r *http.Request, name string) (string, error) {
return "", errors.Errorf("dummy getclaim func cannot retrieve claim '%s'", name)
}
func WithGetClaims(fn GetClaimsFunc) OptionFunc {
func WithGetClaims(fn GetClaimFunc) OptionFunc {
return func(o *Option) {
o.GetClaims = fn
o.GetClaim = fn
}
}
@ -45,3 +47,34 @@ func WithProfileClaims(claims ...string) OptionFunc {
o.ProfileClaims = claims
}
}
func WithJWT(getKeySet jwtutil.GetKeySetFunc) OptionFunc {
funcs := []jwtutil.FindTokenOptionFunc{
jwtutil.WithFinders(
jwtutil.FindTokenFromAuthorizationHeader,
jwtutil.FindTokenFromQueryString(CookieName),
jwtutil.FindTokenFromCookie(CookieName),
),
}
return func(o *Option) {
o.GetClaim = func(ctx context.Context, r *http.Request, name string) (string, error) {
token, err := jwtutil.FindToken(r, getKeySet, funcs...)
if err != nil {
return "", errors.WithStack(err)
}
tokenMap, err := token.AsMap(ctx)
if err != nil {
return "", errors.WithStack(err)
}
value, exists := tokenMap[name]
if !exists {
return "", nil
}
return fmt.Sprintf("%v", value), nil
}
}
}

View File

@ -95,7 +95,7 @@ func (m *Module) writeBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -106,7 +106,7 @@ func (m *Module) writeBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value
defer func() {
if err := writer.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
logger.Error(ctx, "could not close blob writer", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close blob writer", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -129,7 +129,7 @@ func (m *Module) getBlobInfo(call goja.FunctionCall, rt *goja.Runtime) goja.Valu
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -153,7 +153,7 @@ func (m *Module) readBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
defer func() {
if err := reader.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
logger.Error(ctx, "could not close blob reader", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close blob reader", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -245,7 +245,7 @@ func (m *Module) handleMessages() {
res, err := m.handleUploadRequest(uploadRequest)
if err != nil {
logger.Error(ctx, "could not handle upload request", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not handle upload request", logger.CapturedE(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
@ -267,7 +267,7 @@ func (m *Module) handleMessages() {
res, err := m.handleDownloadRequest(downloadRequest)
if err != nil {
logger.Error(ctx, "could not handle download request", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not handle download request", logger.CapturedE(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
@ -354,7 +354,7 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close file", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -365,7 +365,7 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -376,13 +376,13 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close file", logger.CapturedE(errors.WithStack(err)))
}
}()
defer func() {
if err := writer.Close(); err != nil {
logger.Error(ctx, "could not close writer", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not close writer", logger.CapturedE(errors.WithStack(err)))
}
}()
@ -453,7 +453,7 @@ func (m *Module) openBlob(ctx context.Context, bucketName string, blobID storage
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)), logger.F("bucket", bucket))
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)), logger.F("bucket", bucket))
}
}()

View File

@ -1,14 +1,15 @@
package blob
import (
"io/ioutil"
"context"
"os"
"testing"
"cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
@ -27,7 +28,7 @@ func TestBlobModule(t *testing.T) {
ModuleFactory(bus, store),
)
data, err := ioutil.ReadFile("testdata/blob.js")
data, err := os.ReadFile("testdata/blob.js")
if err != nil {
t.Fatal(err)
}
@ -38,7 +39,8 @@ func TestBlobModule(t *testing.T) {
defer server.Stop()
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
}

View File

@ -148,7 +148,7 @@ func SearchDevices(ctx context.Context) (chan Device, error) {
defer searchDevicesMutex.Unlock()
if err := service.Run(ctx, serviceDiscoveryPollingInterval); err != nil && !errors.Is(err, context.DeadlineExceeded) {
logger.Error(ctx, "error while running cast service discovery", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error while running cast service discovery", logger.CapturedE(errors.WithStack(err)))
}
}()

View File

@ -71,7 +71,7 @@ func (d *Service) Run(ctx context.Context, interval time.Duration) error {
logger.Error(
ctx, "could not poll interface",
logger.E(errors.WithStack(err)), logger.F("iface", iface.Name),
logger.CapturedE(errors.WithStack(err)), logger.F("iface", iface.Name),
)
}
}(pollCtx, iface)

View File

@ -60,7 +60,7 @@ func (m *Module) refreshDevices(call goja.FunctionCall, rt *goja.Runtime) goja.V
devices, err := ListDevices(ctx, true)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error refreshing casting devices list", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error refreshing casting devices list", logger.CapturedE(errors.WithStack(err)))
promise.Reject(err)
@ -108,7 +108,7 @@ func (m *Module) loadUrl(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
err := LoadURL(ctx, deviceUUID, url)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error while casting url", logger.E(err))
logger.Error(ctx, "error while casting url", logger.CapturedE(err))
promise.Reject(err)
@ -143,7 +143,7 @@ func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
err := StopCast(ctx, deviceUUID)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error while quitting casting device app", logger.E(errors.WithStack(err)))
logger.Error(ctx, "error while quitting casting device app", logger.CapturedE(errors.WithStack(err)))
promise.Reject(err)
@ -178,7 +178,7 @@ func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value
status, err := getStatus(ctx, deviceUUID)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error while getting casting device status", logger.E(err))
logger.Error(ctx, "error while getting casting device status", logger.CapturedE(err))
promise.Reject(err)

View File

@ -40,7 +40,8 @@ func TestCastModule(t *testing.T) {
t.Fatal(err)
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -74,7 +75,8 @@ func TestCastModuleRefreshDevices(t *testing.T) {
t.Fatal(err)
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}

View File

@ -48,7 +48,7 @@ func (m *Module) handleMessages() {
res, err := m.handleFetchRequest(fetchRequest)
if err != nil {
logger.Error(ctx, "could not handle fetch request", logger.E(errors.WithStack(err)))
logger.Error(ctx, "could not handle fetch request", logger.CapturedE(errors.WithStack(err)))
return nil, errors.WithStack(err)
}

View File

@ -39,7 +39,8 @@ func TestFetchModule(t *testing.T) {
defer server.Stop()
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}

View File

@ -18,30 +18,17 @@ func (m *LifecycleModule) Name() string {
func (m *LifecycleModule) Export(export *goja.Object) {
}
func (m *LifecycleModule) OnInit(rt *goja.Runtime) (err error) {
call, ok := goja.AssertFunction(rt.Get("onInit"))
func (m *LifecycleModule) OnInit(ctx context.Context, rt *goja.Runtime) (err error) {
_, ok := goja.AssertFunction(rt.Get("onInit"))
if !ok {
logger.Warn(context.Background(), "could not find onInit() function")
logger.Warn(ctx, "could not find onInit() function")
return nil
}
defer func() {
if recovered := recover(); recovered != nil {
revoveredErr, ok := recovered.(error)
if ok {
logger.Error(context.Background(), "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
err = errors.WithStack(app.ErUnknownError)
return
}
panic(recovered)
}
}()
call(nil)
if _, err := rt.RunString("setTimeout(onInit, 0)"); err != nil {
return errors.WithStack(err)
}
return nil
}

View File

@ -115,7 +115,7 @@ func (m *Module) handleClientMessages() {
case msg := <-clientMessages:
clientMessage, ok := msg.(*module.ClientMessage)
if !ok {
logger.Error(
logger.Warn(
ctx,
"unexpected message type",
logger.F("message", msg),
@ -138,7 +138,7 @@ func (m *Module) handleClientMessages() {
logger.Error(
ctx,
"on client message error",
logger.E(err),
logger.CapturedE(errors.WithStack(err)),
)
}
}

View File

@ -51,8 +51,8 @@ func (m *RPCModule) Export(export *goja.Object) {
}
}
func (m *RPCModule) OnInit(rt *goja.Runtime) error {
go m.handleMessages()
func (m *RPCModule) OnInit(ctx context.Context, rt *goja.Runtime) error {
go m.handleMessages(ctx)
return nil
}
@ -92,9 +92,7 @@ func (m *RPCModule) unregister(call goja.FunctionCall, rt *goja.Runtime) goja.Va
return nil
}
func (m *RPCModule) handleMessages() {
ctx := context.Background()
func (m *RPCModule) handleMessages(ctx context.Context) {
clientMessages, err := m.bus.Subscribe(ctx, MessageNamespaceClient)
if err != nil {
panic(errors.WithStack(err))
@ -115,7 +113,7 @@ func (m *RPCModule) handleMessages() {
if err := m.sendResponse(ctx, res); err != nil {
logger.Error(
ctx, "could not send response",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("response", res),
logger.F("request", req),
)
@ -149,7 +147,7 @@ func (m *RPCModule) handleMessage(ctx context.Context, msg bus.Message, sendRes
if err := m.sendMethodNotFoundResponse(clientMessage.Context, req); err != nil {
logger.Error(
ctx, "could not send method not found response",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("request", req),
)
}
@ -164,7 +162,7 @@ func (m *RPCModule) handleMessage(ctx context.Context, msg bus.Message, sendRes
if err := m.sendMethodNotFoundResponse(clientMessage.Context, req); err != nil {
logger.Error(
ctx, "could not send method not found response",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("request", req),
)
}
@ -176,14 +174,14 @@ func (m *RPCModule) handleMessage(ctx context.Context, msg bus.Message, sendRes
if err != nil {
logger.Error(
ctx, "rpc call error",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("request", req),
)
if err := m.sendErrorResponse(clientMessage.Context, req, err); err != nil {
logger.Error(
ctx, "could not send error response",
logger.E(errors.WithStack(err)),
logger.CapturedE(errors.WithStack(err)),
logger.F("originalError", err),
logger.F("request", req),
)

View File

@ -5,18 +5,19 @@ import (
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module/util"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/dop251/goja"
"github.com/pkg/errors"
)
const (
AnyType ValueType = "*"
AnyName string = "*"
AnyType share.ValueType = "*"
AnyName string = "*"
)
type Module struct {
appID app.ID
repository Repository
appID app.ID
store share.Store
}
func (m *Module) Name() string {
@ -48,19 +49,19 @@ func (m *Module) Export(export *goja.Object) {
panic(errors.Wrap(err, "could not set 'ANY_NAME' property"))
}
if err := export.Set("TYPE_TEXT", TypeText); err != nil {
if err := export.Set("TYPE_TEXT", share.TypeText); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_TEXT' property"))
}
if err := export.Set("TYPE_NUMBER", TypeNumber); err != nil {
if err := export.Set("TYPE_NUMBER", share.TypeNumber); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_NUMBER' property"))
}
if err := export.Set("TYPE_BOOL", TypeBool); err != nil {
if err := export.Set("TYPE_BOOL", share.TypeBool); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_BOOL' property"))
}
if err := export.Set("TYPE_PATH", TypePath); err != nil {
if err := export.Set("TYPE_PATH", share.TypePath); err != nil {
panic(errors.Wrap(err, "could not set 'TYPE_PATH' property"))
}
}
@ -69,20 +70,20 @@ func (m *Module) upsertResource(call goja.FunctionCall, rt *goja.Runtime) goja.V
ctx := util.AssertContext(call.Argument(0), rt)
resourceID := assertResourceID(call.Argument(1), rt)
var attributes []Attribute
var attributes []share.Attribute
if len(call.Arguments) > 2 {
attributes = assertAttributes(call.Arguments[2:], rt)
} else {
attributes = make([]Attribute, 0)
attributes = make([]share.Attribute, 0)
}
for _, attr := range attributes {
if err := AssertType(attr.Value(), attr.Type()); err != nil {
if err := share.AssertType(attr.Value(), attr.Type()); err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
}
resource, err := m.repository.UpdateAttributes(ctx, m.appID, resourceID, attributes...)
resource, err := m.store.UpdateAttributes(ctx, m.appID, resourceID, attributes...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
@ -101,7 +102,7 @@ func (m *Module) deleteAttributes(call goja.FunctionCall, rt *goja.Runtime) goja
names = make([]string, 0)
}
err := m.repository.DeleteAttributes(ctx, m.appID, resourceID, names...)
err := m.store.DeleteAttributes(ctx, m.appID, resourceID, names...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
@ -112,23 +113,23 @@ func (m *Module) deleteAttributes(call goja.FunctionCall, rt *goja.Runtime) goja
func (m *Module) findResources(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
funcs := make([]FindResourcesOptionFunc, 0)
funcs := make([]share.FindResourcesOptionFunc, 0)
if len(call.Arguments) > 1 {
name := util.AssertString(call.Argument(1), rt)
if name != AnyName {
funcs = append(funcs, WithName(name))
funcs = append(funcs, share.WithName(name))
}
}
if len(call.Arguments) > 2 {
valueType := assertValueType(call.Argument(2), rt)
if valueType != AnyType {
funcs = append(funcs, WithType(valueType))
funcs = append(funcs, share.WithType(valueType))
}
}
resources, err := m.repository.FindResources(ctx, funcs...)
resources, err := m.store.FindResources(ctx, funcs...)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
@ -140,7 +141,7 @@ func (m *Module) deleteResource(call goja.FunctionCall, rt *goja.Runtime) goja.V
ctx := util.AssertContext(call.Argument(0), rt)
resourceID := assertResourceID(call.Argument(1), rt)
err := m.repository.DeleteResource(ctx, m.appID, resourceID)
err := m.store.DeleteResource(ctx, m.appID, resourceID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
@ -148,29 +149,29 @@ func (m *Module) deleteResource(call goja.FunctionCall, rt *goja.Runtime) goja.V
return nil
}
func ModuleFactory(appID app.ID, repository Repository) app.ServerModuleFactory {
func ModuleFactory(appID app.ID, store share.Store) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
return &Module{
appID: appID,
repository: repository,
appID: appID,
store: store,
}
}
}
func assertResourceID(v goja.Value, r *goja.Runtime) ResourceID {
func assertResourceID(v goja.Value, r *goja.Runtime) share.ResourceID {
value := v.Export()
switch typ := value.(type) {
case string:
return ResourceID(typ)
case ResourceID:
return share.ResourceID(typ)
case share.ResourceID:
return typ
default:
panic(r.ToValue(errors.Errorf("expected value to be a string or ResourceID, got '%T'", value)))
}
}
func assertAttributes(values []goja.Value, r *goja.Runtime) []Attribute {
attributes := make([]Attribute, len(values))
func assertAttributes(values []goja.Value, r *goja.Runtime) []share.Attribute {
attributes := make([]share.Attribute, len(values))
for idx, val := range values {
export := val.Export()
@ -195,12 +196,12 @@ func assertAttributes(values []goja.Value, r *goja.Runtime) []Attribute {
panic(r.ToValue(errors.Errorf("could not find 'type' property on attribute '%v'", export)))
}
var valueType ValueType
var valueType share.ValueType
switch typ := rawType.(type) {
case ValueType:
case share.ValueType:
valueType = typ
case string:
valueType = ValueType(typ)
valueType = share.ValueType(typ)
default:
panic(r.ToValue(errors.Errorf("unexpected value for attribute property 'type': expected 'string' or 'ValueType', got '%T'", rawType)))
@ -211,7 +212,7 @@ func assertAttributes(values []goja.Value, r *goja.Runtime) []Attribute {
panic(r.ToValue(errors.Errorf("could not find 'value' property on attribute '%v'", export)))
}
attributes[idx] = NewBaseAttribute(
attributes[idx] = share.NewBaseAttribute(
name,
valueType,
value,
@ -232,12 +233,12 @@ func assertStrings(values []goja.Value, r *goja.Runtime) []string {
return strings
}
func assertValueType(v goja.Value, r *goja.Runtime) ValueType {
func assertValueType(v goja.Value, r *goja.Runtime) share.ValueType {
value := v.Export()
switch typ := value.(type) {
case string:
return ValueType(typ)
case ValueType:
return share.ValueType(typ)
case share.ValueType:
return typ
default:
panic(r.ToValue(errors.Errorf("expected value to be a string or ValueType, got '%T'", value)))
@ -245,7 +246,7 @@ func assertValueType(v goja.Value, r *goja.Runtime) ValueType {
}
type gojaResource struct {
ID ResourceID `goja:"id" json:"id"`
ID share.ResourceID `goja:"id" json:"id"`
Origin app.ID `goja:"origin" json:"origin"`
Attributes []*gojaAttribute `goja:"attributes" json:"attributes"`
}
@ -254,7 +255,7 @@ func (r *gojaResource) Has(call goja.FunctionCall, rt *goja.Runtime) goja.Value
name := util.AssertString(call.Argument(0), rt)
valueType := assertValueType(call.Argument(1), rt)
hasAttr := HasAttribute(toResource(r), name, valueType)
hasAttr := share.HasAttribute(toResource(r), name, valueType)
return rt.ToValue(hasAttr)
}
@ -268,7 +269,7 @@ func (r *gojaResource) Get(call goja.FunctionCall, rt *goja.Runtime) goja.Value
defaultValue = call.Argument(2).Export()
}
attr := GetAttribute(toResource(r), name, valueType)
attr := share.GetAttribute(toResource(r), name, valueType)
if attr == nil {
return rt.ToValue(defaultValue)
@ -278,14 +279,14 @@ func (r *gojaResource) Get(call goja.FunctionCall, rt *goja.Runtime) goja.Value
}
type gojaAttribute struct {
Name string `goja:"name" json:"name"`
Type ValueType `goja:"type" json:"type"`
Value any `goja:"value" json:"value"`
CreatedAt time.Time `goja:"createdAt" json:"createdAt"`
UpdatedAt time.Time `goja:"updatedAt" json:"updatedAt"`
Name string `goja:"name" json:"name"`
Type share.ValueType `goja:"type" json:"type"`
Value any `goja:"value" json:"value"`
CreatedAt time.Time `goja:"createdAt" json:"createdAt"`
UpdatedAt time.Time `goja:"updatedAt" json:"updatedAt"`
}
func toGojaResource(res Resource) *gojaResource {
func toGojaResource(res share.Resource) *gojaResource {
attributes := make([]*gojaAttribute, len(res.Attributes()))
for idx, attr := range res.Attributes() {
@ -305,7 +306,7 @@ func toGojaResource(res Resource) *gojaResource {
}
}
func toGojaResources(resources []Resource) []*gojaResource {
func toGojaResources(resources []share.Resource) []*gojaResource {
gojaResources := make([]*gojaResource, len(resources))
for idx, res := range resources {
gojaResources[idx] = toGojaResource(res)
@ -313,19 +314,19 @@ func toGojaResources(resources []Resource) []*gojaResource {
return gojaResources
}
func toResource(res *gojaResource) Resource {
return NewBaseResource(
func toResource(res *gojaResource) share.Resource {
return share.NewBaseResource(
res.Origin,
res.ID,
toAttributes(res.Attributes)...,
)
}
func toAttributes(gojaAttributes []*gojaAttribute) []Attribute {
attributes := make([]Attribute, len(gojaAttributes))
func toAttributes(gojaAttributes []*gojaAttribute) []share.Attribute {
attributes := make([]share.Attribute, len(gojaAttributes))
for idx, gojaAttr := range gojaAttributes {
attr := NewBaseAttribute(
attr := share.NewBaseAttribute(
gojaAttr.Name,
gojaAttr.Type,
gojaAttr.Value,

View File

@ -1,21 +1,23 @@
package testsuite
package share
import (
"context"
"io/fs"
"os"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/module/share"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
)
func TestModule(t *testing.T, newRepo NewTestRepoFunc) {
func TestModule(t *testing.T) {
logger.SetLevel(logger.LevelDebug)
repo, err := newRepo("module")
store, err := driver.NewShareStore("sqlite://testdata/test_share_module.sqlite")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -23,10 +25,10 @@ func TestModule(t *testing.T, newRepo NewTestRepoFunc) {
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
share.ModuleFactory("test.app.edge", repo),
ModuleFactory("test.app.edge", store),
)
data, err := fs.ReadFile(testData, "testdata/share.js")
data, err := os.ReadFile("testdata/share.js")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -35,7 +37,8 @@ func TestModule(t *testing.T, newRepo NewTestRepoFunc) {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}

View File

@ -1,13 +0,0 @@
package sqlite
import (
"testing"
"forge.cadoles.com/arcad/edge/pkg/module/share/testsuite"
"gitlab.com/wpetit/goweb/logger"
)
func TestModule(t *testing.T) {
logger.SetLevel(logger.LevelDebug)
testsuite.TestModule(t, newTestRepo)
}

View File

@ -1 +0,0 @@
*.sqlite*

View File

@ -1,16 +0,0 @@
package testsuite
import (
"testing"
"forge.cadoles.com/arcad/edge/pkg/module/share"
)
type NewTestRepoFunc func(testname string) (share.Repository, error)
func TestRepository(t *testing.T, newRepo NewTestRepoFunc) {
t.Run("Cases", func(t *testing.T) {
t.Parallel()
runRepositoryTests(t, newRepo)
})
}

View File

@ -2,12 +2,12 @@ package store
import (
"context"
"io/ioutil"
"os"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
@ -22,7 +22,7 @@ func TestStoreModule(t *testing.T) {
ModuleFactory(store),
)
data, err := ioutil.ReadFile("testdata/store.js")
data, err := os.ReadFile("testdata/store.js")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
@ -31,7 +31,8 @@ func TestStoreModule(t *testing.T) {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Start(); err != nil {
ctx := context.Background()
if err := server.Start(ctx); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -5,5 +5,6 @@ import LoginIcon from './login.svg';
import HomeIcon from './home.svg';
import LinkIcon from './link.svg';
import LogoutIcon from './logout.svg';
import LoaderIcon from './loader.svg';
export { UserCircleIcon, MenuIcon, CloudIcon, LoginIcon, HomeIcon, LinkIcon, LogoutIcon }
export { LoaderIcon, UserCircleIcon, MenuIcon, CloudIcon, LoginIcon, HomeIcon, LinkIcon, LogoutIcon }

View File

@ -0,0 +1 @@
<svg viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="#000000"><g id="SVGRepo_bgCarrier" stroke-width="0"></g><g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"></g><g id="SVGRepo_iconCarrier"> <g> <path fill="none" d="M0 0h24v24H0z"></path> <path d="M12 2a1 1 0 0 1 1 1v3a1 1 0 0 1-2 0V3a1 1 0 0 1 1-1zm0 15a1 1 0 0 1 1 1v3a1 1 0 0 1-2 0v-3a1 1 0 0 1 1-1zm8.66-10a1 1 0 0 1-.366 1.366l-2.598 1.5a1 1 0 1 1-1-1.732l2.598-1.5A1 1 0 0 1 20.66 7zM7.67 14.5a1 1 0 0 1-.366 1.366l-2.598 1.5a1 1 0 1 1-1-1.732l2.598-1.5a1 1 0 0 1 1.366.366zM20.66 17a1 1 0 0 1-1.366.366l-2.598-1.5a1 1 0 0 1 1-1.732l2.598 1.5A1 1 0 0 1 20.66 17zM7.67 9.5a1 1 0 0 1-1.366.366l-2.598-1.5a1 1 0 1 1 1-1.732l2.598 1.5A1 1 0 0 1 7.67 9.5z"></path> </g> </g></svg>

After

Width:  |  Height:  |  Size: 773 B

View File

@ -0,0 +1,57 @@
import { LitElement, html, css } from 'lit';
import { LoaderIcon } from './icons';
export class Loader extends LitElement {
static styles = css`
:host {
display: inline-block;
height: 100%;
width: 100%;
border-bottom: 1px solid rgb(229,231,235);
border-top: 10px solid transparent;
background-color: #fff;
min-height: 50px;
padding: 10px 0;
}
.container {
display: flex;
align-items: center;
flex-direction: column;
justify-content: center;
font-family: Arial, Helvetica Neue, Helvetica, sans-serif;
font-size: 14px;
color: black;
}
.icon {
height: 35px;
animation-duration: 3s;
animation-name: spin;
animation-iteration-count: infinite;
}
@keyframes spin {
from {
transform: rotateZ(0deg);
}
to {
transform: rotateZ(360deg);
}
}
`;
constructor() {
super();
}
render() {
return html`
<div class="container">
<img class="icon" src="${LoaderIcon}" />
Chargement en cours
</div>
`
}
}

Some files were not shown because too many files have changed in this diff Show More