Compare commits
1 Commits
2024.1.10-
...
2023.10.3-
| Author | SHA1 | Date | |
|---|---|---|---|
| b2f100f987 |
@ -1,4 +1,4 @@
|
||||
RUN_APP_ARGS=""
|
||||
#EDGE_DOCUMENTSTORE_DSN="rpc://localhost:3001/documentstore?tenant=local&appId=%APPID%"
|
||||
#EDGE_BLOBSTORE_DSN="cache://localhost:3001/blobstore?driver=rpc&tenant=local&appId=%APPID%&blobCacheStoreType=fs&blobCacheStoreBaseDir=data/cache/%APPID%&blobCacheSize=64MB"
|
||||
#EDGE_BLOBSTORE_DSN="rpc://localhost:3001/blobstore?tenant=local&appId=%APPID%"
|
||||
#EDGE_SHARESTORE_DSN="rpc://localhost:3001/sharestore?tenant=local"
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,7 +2,7 @@
|
||||
/bin
|
||||
/.env
|
||||
/tools
|
||||
*.sqlite*
|
||||
*.sqlite
|
||||
/.gitea-release
|
||||
/.edge
|
||||
/data
|
||||
|
||||
@ -108,9 +108,6 @@ nfpms:
|
||||
file_info:
|
||||
mode: 0640
|
||||
packager: apk
|
||||
- src: misc/packaging/openrc/storage-server.logrotate.conf
|
||||
dst: /etc/logrotate.d/storage-server
|
||||
packager: apk
|
||||
- dst: /var/lib/storage-server
|
||||
type: dir
|
||||
file_info:
|
||||
@ -119,6 +116,7 @@ nfpms:
|
||||
- dst: /var/log/storage-server
|
||||
type: dir
|
||||
file_info:
|
||||
mode: 0700
|
||||
mode: 0750
|
||||
packager: apk
|
||||
scripts:
|
||||
postinstall: "misc/packaging/common/postinstall-storage-server.sh"
|
||||
2
Jenkinsfile
vendored
2
Jenkinsfile
vendored
@ -35,7 +35,7 @@ pipeline {
|
||||
])
|
||||
]) {
|
||||
sh 'make .mktools'
|
||||
sh "export MKT_PROJECT_VERSION_BRANCH_NAME=${env.BRANCH_NAME}; make gitea-release"
|
||||
sh "MKT_PROJECT_VERSION_BRANCH_NAME=${env.BRANCH_NAME} make gitea-release"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,146 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"flag"
|
||||
"io"
|
||||
mrand "math/rand"
|
||||
"runtime"
|
||||
"time"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
|
||||
)
|
||||
|
||||
var (
|
||||
dsn string
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.StringVar(&dsn, "dsn", "cache://./test-cache.sqlite?driver=sqlite&_pragma=foreign_keys(1)&_pragma=journal_mode=wal&bigCacheShards=32&bigCacheHardMaxCacheSize=128&bigCacheMaxEntrySize=125&bigCacheMaxEntriesInWindow=200000", "blobstore dsn")
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
logger.SetLevel(logger.LevelDebug)
|
||||
|
||||
blobStore, err := driver.NewBlobStore(dsn)
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not create blobstore", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
bucket, err := blobStore.OpenBucket(ctx, "default")
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not open bucket", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := bucket.Close(); err != nil {
|
||||
logger.Fatal(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
go readRandomBlobs(ctx, bucket)
|
||||
|
||||
for {
|
||||
writeRandomBlob(ctx, bucket)
|
||||
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
size, err := bucket.Size(ctx)
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not retrieve bucket size", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "bucket stats", logger.F("size", size))
|
||||
}
|
||||
}
|
||||
|
||||
func readRandomBlobs(ctx context.Context, bucket storage.BlobBucket) {
|
||||
for {
|
||||
infos, err := bucket.List(ctx)
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not list blobs", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
total := len(infos)
|
||||
if total == 0 {
|
||||
logger.Debug(ctx, "no blob yet")
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
blob := infos[mrand.Intn(total)]
|
||||
|
||||
readBlob(ctx, bucket, blob.ID())
|
||||
|
||||
time.Sleep(250 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func readBlob(ctx context.Context, bucket storage.BlobBucket, blobID storage.BlobID) {
|
||||
ctx = logger.With(ctx, logger.F("blobID", blobID))
|
||||
|
||||
reader, err := bucket.NewReader(ctx, blobID)
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not create reader", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
logger.Fatal(ctx, "could not close reader", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := io.ReadAll(reader); err != nil {
|
||||
logger.Fatal(ctx, "could not read blob", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}
|
||||
|
||||
func writeRandomBlob(ctx context.Context, bucket storage.BlobBucket) {
|
||||
blobID := storage.NewBlobID()
|
||||
buff := make([]byte, 10*1024)
|
||||
|
||||
writer, err := bucket.NewWriter(ctx, blobID)
|
||||
if err != nil {
|
||||
logger.Fatal(ctx, "could not create writer", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := writer.Close(); err != nil {
|
||||
logger.Fatal(ctx, "could not close writer", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := rand.Read(buff); err != nil {
|
||||
logger.Fatal(ctx, "could not read random data", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
if _, err := writer.Write(buff); err != nil {
|
||||
logger.Fatal(ctx, "could not write blob", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
printMemUsage(ctx)
|
||||
}
|
||||
|
||||
func printMemUsage(ctx context.Context) {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
logger.Debug(
|
||||
ctx, "memory usage",
|
||||
logger.F("alloc", m.Alloc/1024/1024),
|
||||
logger.F("totalAlloc", m.TotalAlloc/1024/1024),
|
||||
logger.F("sys", m.Sys/1024/1024),
|
||||
logger.F("numGC", m.NumGC),
|
||||
)
|
||||
}
|
||||
@ -1,56 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bundle"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/urfave/cli/v2"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
func InfoCommand() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "info",
|
||||
Usage: "Print app manifest informations",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "path",
|
||||
Usage: "use `PATH` as app bundle (zip, zim or directory bundle)",
|
||||
Aliases: []string{"p"},
|
||||
Value: "",
|
||||
Required: true,
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
appPath := ctx.String("path")
|
||||
|
||||
bundle, err := bundle.FromPath(appPath)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not load app bundle")
|
||||
}
|
||||
|
||||
manifest, err := app.LoadManifest(bundle)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not load app manifest")
|
||||
}
|
||||
|
||||
if valid, err := manifest.Validate(manifestMetadataValidators...); !valid {
|
||||
return errors.Wrap(err, "invalid app manifest")
|
||||
}
|
||||
|
||||
encoder := yaml.NewEncoder(os.Stdout)
|
||||
|
||||
if err := encoder.Encode(manifest); err != nil {
|
||||
return errors.Wrap(err, "could not encode manifest")
|
||||
}
|
||||
|
||||
if err := encoder.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -12,7 +12,6 @@ func Root() *cli.Command {
|
||||
RunCommand(),
|
||||
PackageCommand(),
|
||||
HashPasswordCommand(),
|
||||
InfoCommand(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -23,11 +23,10 @@ import (
|
||||
authModule "forge.cadoles.com/arcad/edge/pkg/module/auth"
|
||||
authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
|
||||
authModuleMiddleware "forge.cadoles.com/arcad/edge/pkg/module/auth/middleware"
|
||||
blobModule "forge.cadoles.com/arcad/edge/pkg/module/blob"
|
||||
castModule "forge.cadoles.com/arcad/edge/pkg/module/cast"
|
||||
fetchModule "forge.cadoles.com/arcad/edge/pkg/module/fetch"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/blob"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/cast"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
|
||||
netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
|
||||
rpcModule "forge.cadoles.com/arcad/edge/pkg/module/rpc"
|
||||
shareModule "forge.cadoles.com/arcad/edge/pkg/module/share"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
@ -45,13 +44,10 @@ import (
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/argon2id"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/module/auth/http/passwd/plain"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
|
||||
|
||||
// Register storage drivers
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/share"
|
||||
)
|
||||
|
||||
@ -107,11 +103,6 @@ func RunCommand() *cli.Command {
|
||||
Usage: "use `FILE` as local accounts",
|
||||
Value: ".edge/%APPID%/accounts.json",
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
Name: "max-upload-size",
|
||||
Usage: "use `MAX-UPLOAD-SIZE` as blob max upload size",
|
||||
Value: 128 << (10 * 2), // 128Mb
|
||||
},
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
address := ctx.String("address")
|
||||
@ -123,7 +114,6 @@ func RunCommand() *cli.Command {
|
||||
documentstoreDSN := ctx.String("documentstore-dsn")
|
||||
shareStoreDSN := ctx.String("sharestore-dsn")
|
||||
accountsFile := ctx.String("accounts-file")
|
||||
maxUploadSize := ctx.Int64("max-upload-size")
|
||||
|
||||
logger.SetFormat(logger.Format(logFormat))
|
||||
logger.SetLevel(logger.Level(logLevel))
|
||||
@ -169,8 +159,8 @@ func RunCommand() *cli.Command {
|
||||
|
||||
appCtx := logger.With(cmdCtx, logger.F("address", address))
|
||||
|
||||
if err := runApp(appCtx, path, address, documentstoreDSN, blobstoreDSN, shareStoreDSN, accountsFile, appsRepository, maxUploadSize); err != nil {
|
||||
logger.Error(appCtx, "could not run app", logger.CapturedE(errors.WithStack(err)))
|
||||
if err := runApp(appCtx, path, address, documentstoreDSN, blobstoreDSN, shareStoreDSN, accountsFile, appsRepository); err != nil {
|
||||
logger.Error(appCtx, "could not run app", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}(p, port, idx)
|
||||
}
|
||||
@ -182,7 +172,7 @@ func RunCommand() *cli.Command {
|
||||
}
|
||||
}
|
||||
|
||||
func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN, shareStoreDSN, accountsFile string, appRepository appModule.Repository, maxUploadSize int64) error {
|
||||
func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN, shareStoreDSN, accountsFile string, appRepository appModule.Repository) error {
|
||||
absPath, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "could not resolve path '%s'", path)
|
||||
@ -243,14 +233,12 @@ func runApp(ctx context.Context, path, address, documentStoreDSN, blobStoreDSN,
|
||||
return jwtutil.NewSymmetricKeySet(dummySecret)
|
||||
}),
|
||||
),
|
||||
blobModule.Mount(maxUploadSize), // 10Mb,
|
||||
fetchModule.Mount(),
|
||||
),
|
||||
appHTTP.WithHTTPMiddlewares(
|
||||
authModuleMiddleware.AnonymousUser(key, jwa.HS256),
|
||||
),
|
||||
)
|
||||
if err := handler.Load(ctx, bundle); err != nil {
|
||||
if err := handler.Load(bundle); err != nil {
|
||||
return errors.Wrap(err, "could not load app bundle")
|
||||
}
|
||||
|
||||
@ -287,18 +275,18 @@ func getServerModules(deps *moduleDeps) []app.ServerModuleFactory {
|
||||
module.LifecycleModuleFactory(),
|
||||
module.ContextModuleFactory(),
|
||||
module.ConsoleModuleFactory(),
|
||||
castModule.CastModuleFactory(),
|
||||
cast.CastModuleFactory(),
|
||||
netModule.ModuleFactory(deps.Bus),
|
||||
rpcModule.ModuleFactory(deps.Bus),
|
||||
module.RPCModuleFactory(deps.Bus),
|
||||
module.StoreModuleFactory(deps.DocumentStore),
|
||||
blobModule.ModuleFactory(deps.Bus, deps.BlobStore),
|
||||
blob.ModuleFactory(deps.Bus, deps.BlobStore),
|
||||
authModule.ModuleFactory(
|
||||
authModule.WithJWT(func() (jwk.Set, error) {
|
||||
return jwtutil.NewSymmetricKeySet(dummySecret)
|
||||
}),
|
||||
),
|
||||
appModule.ModuleFactory(deps.AppRepository),
|
||||
fetchModule.ModuleFactory(deps.Bus),
|
||||
fetch.ModuleFactory(deps.Bus),
|
||||
shareModule.ModuleFactory(deps.AppID, deps.ShareStore),
|
||||
}
|
||||
}
|
||||
@ -366,7 +354,7 @@ func findMatchingDeviceAddress(ctx context.Context, from string, defaultAddr str
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not retrieve iface adresses",
|
||||
logger.CapturedE(errors.WithStack(err)), logger.F("iface", ifa.Name),
|
||||
logger.E(errors.WithStack(err)), logger.F("iface", ifa.Name),
|
||||
)
|
||||
|
||||
continue
|
||||
@ -377,7 +365,7 @@ func findMatchingDeviceAddress(ctx context.Context, from string, defaultAddr str
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not parse address",
|
||||
logger.CapturedE(errors.WithStack(err)), logger.F("address", addr.String()),
|
||||
logger.E(errors.WithStack(err)), logger.F("address", addr.String()),
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/flag"
|
||||
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
|
||||
"github.com/lestrrat-go/jwx/v2/jwa"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/urfave/cli/v2"
|
||||
)
|
||||
|
||||
func CheckToken() *cli.Command {
|
||||
return &cli.Command{
|
||||
Name: "check-token",
|
||||
Usage: "Validate and print the given token with the private key",
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "token",
|
||||
Required: true,
|
||||
},
|
||||
flag.PrivateKey,
|
||||
flag.PrivateKeySigningAlgorithm,
|
||||
flag.PrivateKeyDefaultSize,
|
||||
},
|
||||
Action: func(ctx *cli.Context) error {
|
||||
privateKeyFile := flag.GetPrivateKey(ctx)
|
||||
signingAlgorithm := flag.GetSigningAlgorithm(ctx)
|
||||
privateKeyDefaultSize := flag.GetPrivateKeyDefaultSize(ctx)
|
||||
rawToken := ctx.String("token")
|
||||
|
||||
if rawToken == "" {
|
||||
return errors.New("you must provide a value for --token flag")
|
||||
}
|
||||
|
||||
privateKey, err := jwtutil.LoadOrGenerateKey(
|
||||
privateKeyFile,
|
||||
privateKeyDefaultSize,
|
||||
)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
keySet, err := jwtutil.NewKeySet()
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
err = jwtutil.AddKeyWithSigningAlgo(keySet, privateKey, jwa.SignatureAlgorithm(signingAlgorithm))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
token, err := jwtutil.Parse([]byte(rawToken), keySet)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
claims, err := token.AsMap(ctx.Context)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
json, err := json.MarshalIndent(claims, "", " ")
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
fmt.Println(string(json))
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -10,7 +10,6 @@ func Root() *cli.Command {
|
||||
Usage: "Auth related command",
|
||||
Subcommands: []*cli.Command{
|
||||
NewToken(),
|
||||
CheckToken(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,12 +4,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/getsentry/sentry-go"
|
||||
"github.com/hashicorp/golang-lru/v2/expirable"
|
||||
"github.com/keegancsmith/rpc"
|
||||
"github.com/lestrrat-go/jwx/v2/jwa"
|
||||
@ -22,15 +20,13 @@ import (
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
// Register storage drivers
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/cmd/storage-server/command/flag"
|
||||
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
|
||||
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage/share"
|
||||
)
|
||||
|
||||
@ -45,35 +41,20 @@ func Run() *cli.Command {
|
||||
Aliases: []string{"addr"},
|
||||
Value: ":3001",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "log-level",
|
||||
EnvVars: []string{"STORAGE_SERVER_LOG_LEVEL"},
|
||||
Value: int(logger.LevelError),
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "blobstore-dsn-pattern",
|
||||
EnvVars: []string{"STORAGE_SERVER_BLOBSTORE_DSN_PATTERN"},
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/blobstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "documentstore-dsn-pattern",
|
||||
EnvVars: []string{"STORAGE_SERVER_DOCUMENTSTORE_DSN_PATTERN"},
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/%%APPID%%/documentstore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sharestore-dsn-pattern",
|
||||
EnvVars: []string{"STORAGE_SERVER_SHARESTORE_DSN_PATTERN"},
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d&_pragma=journal_mode=wal", (60 * time.Second).Milliseconds()),
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sentry-dsn",
|
||||
EnvVars: []string{"STORAGE_SERVER_SENTRY_DSN"},
|
||||
Value: "",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "sentry-environment",
|
||||
EnvVars: []string{"STORAGE_SERVER_SENTRY_ENVIRONMENT"},
|
||||
Value: "",
|
||||
Value: fmt.Sprintf("sqlite://data/%%TENANT%%/sharestore.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", (60 * time.Second).Milliseconds()),
|
||||
},
|
||||
flag.PrivateKey,
|
||||
flag.PrivateKeySigningAlgorithm,
|
||||
@ -99,33 +80,6 @@ func Run() *cli.Command {
|
||||
privateKeyFile := flag.GetPrivateKey(ctx)
|
||||
signingAlgorithm := flag.GetSigningAlgorithm(ctx)
|
||||
privateKeyDefaultSize := flag.GetPrivateKeyDefaultSize(ctx)
|
||||
logLevel := ctx.Int("log-level")
|
||||
|
||||
logger.SetLevel(logger.Level(logLevel))
|
||||
|
||||
sentryDSN := ctx.String("sentry-dsn")
|
||||
sentryEnvironment := ctx.String("sentry-environment")
|
||||
if sentryDSN != "" {
|
||||
if sentryEnvironment == "" {
|
||||
sentryEnvironment, _ = os.Hostname()
|
||||
}
|
||||
|
||||
err := sentry.Init(sentry.ClientOptions{
|
||||
Dsn: sentryDSN,
|
||||
Debug: logLevel == int(logger.LevelDebug),
|
||||
AttachStacktrace: true,
|
||||
Environment: sentryEnvironment,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx.Context, "could not initialize sentry", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
logger.SetCaptureFunc(func(err error) {
|
||||
sentry.CaptureException(err)
|
||||
})
|
||||
|
||||
defer sentry.Flush(2 * time.Second)
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
|
||||
@ -137,6 +91,11 @@ func Run() *cli.Command {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
publicKey, err := privateKey.PublicKey()
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
getBlobStoreServer := createGetCachedStoreServer(
|
||||
func(dsn string) (storage.BlobStore, error) {
|
||||
return driver.NewBlobStore(dsn)
|
||||
@ -166,17 +125,12 @@ func Run() *cli.Command {
|
||||
|
||||
router.Use(middleware.RealIP)
|
||||
router.Use(middleware.Logger)
|
||||
|
||||
logger.Debug(ctx.Context, "using authentication", logger.F("privateKey", privateKeyFile), logger.F("signingAlgorithm", signingAlgorithm))
|
||||
|
||||
router.Use(authenticate(privateKey, jwa.SignatureAlgorithm(signingAlgorithm)))
|
||||
router.Use(authenticate(publicKey, jwa.SignatureAlgorithm(signingAlgorithm)))
|
||||
|
||||
router.Handle("/blobstore", createStoreHandler(getBlobStoreServer, blobStoreDSNPattern, true, cacheSize, cacheTTL))
|
||||
router.Handle("/documentstore", createStoreHandler(getDocumentStoreServer, documentStoreDSNPattern, true, cacheSize, cacheTTL))
|
||||
router.Handle("/sharestore", createStoreHandler(getShareStoreServer, shareStoreDSNPattern, false, cacheSize, cacheTTL))
|
||||
|
||||
logger.Info(ctx.Context, "listening", logger.F("addr", addr))
|
||||
|
||||
if err := http.ListenAndServe(addr, router); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
@ -242,7 +196,7 @@ func createStoreHandler(getStoreServer getRPCServerFunc, dsnPattern string, appI
|
||||
|
||||
server, err := getStoreServer(cacheSize, cacheTTL, tenant, appID, dsnPattern)
|
||||
if err != nil {
|
||||
logger.Error(r.Context(), "could not retrieve store server", logger.CapturedE(errors.WithStack(err)), logger.F("tenant", tenant))
|
||||
logger.Error(r.Context(), "could not retrieve store server", logger.E(errors.WithStack(err)), logger.F("tenant", tenant))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -264,17 +218,15 @@ func authenticate(privateKey jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) f
|
||||
ctx := r.Context()
|
||||
|
||||
createKeySet.Do(func() {
|
||||
var keySet jwk.Set
|
||||
|
||||
keySet, err = jwtutil.NewKeySet()
|
||||
err = privateKey.Set(jwk.AlgorithmKey, signingAlgorithm)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = jwtutil.AddKeyWithSigningAlgo(keySet, privateKey, signingAlgorithm)
|
||||
var keySet jwk.Set
|
||||
|
||||
keySet, err = jwtutil.NewKeySet(privateKey)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
@ -283,7 +235,7 @@ func authenticate(privateKey jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) f
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not create keyset accessor", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not create keyset accessor", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -293,7 +245,7 @@ func authenticate(privateKey jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) f
|
||||
jwtutil.FindTokenFromQueryString("token"),
|
||||
))
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not find jwt token", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not find jwt token", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
|
||||
|
||||
return
|
||||
@ -301,7 +253,7 @@ func authenticate(privateKey jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) f
|
||||
|
||||
tokenMap, err := token.AsMap(ctx)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not transform token to map", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not transform token to map", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
|
||||
|
||||
return
|
||||
|
||||
36
go.mod
36
go.mod
@ -1,37 +1,29 @@
|
||||
module forge.cadoles.com/arcad/edge
|
||||
|
||||
go 1.21
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/getsentry/sentry-go v0.25.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6
|
||||
github.com/hashicorp/mdns v1.0.5
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
|
||||
github.com/jackc/puddle/v2 v2.2.1
|
||||
github.com/keegancsmith/rpc v1.3.0
|
||||
github.com/klauspost/compress v1.16.6
|
||||
github.com/lestrrat-go/jwx/v2 v2.0.8
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2
|
||||
github.com/ulikunitz/xz v0.5.11
|
||||
go.uber.org/goleak v1.3.0
|
||||
modernc.org/sqlite v1.20.4
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.75.0 // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.0 // indirect
|
||||
github.com/go-playground/locales v0.12.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.16.0 // indirect
|
||||
github.com/goccy/go-json v0.9.11 // indirect
|
||||
github.com/gogo/protobuf v0.0.0-20161014173244-50d1bd39ce4e // indirect
|
||||
github.com/leodido/go-urn v1.2.1 // indirect
|
||||
github.com/leodido/go-urn v1.1.0 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httprc v1.0.4 // indirect
|
||||
github.com/lestrrat-go/iter v1.0.2 // indirect
|
||||
github.com/lestrrat-go/option v1.0.0 // indirect
|
||||
github.com/miekg/dns v1.1.53 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20210226172003-ab064af71705 // indirect
|
||||
google.golang.org/grpc v1.35.0 // indirect
|
||||
gopkg.in/go-playground/validator.v9 v9.29.1 // indirect
|
||||
@ -57,8 +49,8 @@ require (
|
||||
github.com/gorilla/websocket v1.4.2 // indirect
|
||||
github.com/igm/sockjs-go/v3 v3.0.2
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-colorable v0.1.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/orcaman/concurrent-map v1.0.0
|
||||
@ -67,14 +59,14 @@ require (
|
||||
github.com/russross/blackfriday/v2 v2.1.0 // indirect
|
||||
github.com/urfave/cli/v2 v2.24.3
|
||||
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
|
||||
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648
|
||||
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b
|
||||
go.opencensus.io v0.22.5 // indirect
|
||||
golang.org/x/crypto v0.10.0
|
||||
golang.org/x/crypto v0.7.0
|
||||
golang.org/x/mod v0.10.0
|
||||
golang.org/x/net v0.11.0
|
||||
golang.org/x/sys v0.9.0 // indirect
|
||||
golang.org/x/term v0.9.0 // indirect
|
||||
golang.org/x/text v0.10.0 // indirect
|
||||
golang.org/x/net v0.9.0 // indirect
|
||||
golang.org/x/sys v0.7.0 // indirect
|
||||
golang.org/x/term v0.7.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
golang.org/x/tools v0.8.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
@ -88,5 +80,3 @@ require (
|
||||
modernc.org/strutil v1.1.3 // indirect
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
replace github.com/allegro/bigcache/v3 v3.1.0 => github.com/Bornholm/bigcache v0.0.0-20231201111725-1ddf51584cad
|
||||
|
||||
99
go.sum
99
go.sum
@ -101,20 +101,16 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
|
||||
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
|
||||
github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI=
|
||||
github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
|
||||
github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ=
|
||||
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
|
||||
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
|
||||
github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
|
||||
github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
|
||||
github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
|
||||
github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
|
||||
github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
|
||||
github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
|
||||
github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
|
||||
github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
|
||||
@ -147,10 +143,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
@ -163,9 +157,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
|
||||
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
@ -179,8 +171,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf
|
||||
github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
|
||||
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
@ -196,8 +188,8 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
|
||||
github.com/hashicorp/go.net v0.0.0-20151006203346-104dcad90073/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
|
||||
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
||||
github.com/hashicorp/mdns v0.0.0-20151206042412-9d85cf22f9f8/go.mod h1:aa76Av3qgPeIQp9Y3qIkTBPieQYNkQ13Kxe7pze9Wb0=
|
||||
github.com/hashicorp/mdns v1.0.5 h1:1M5hW1cunYeoXOqHwEb/GBDDHAFo0Yqb/uz/beC6LbE=
|
||||
@ -206,10 +198,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/igm/sockjs-go/v3 v3.0.2 h1:2m0k53w0DBiGozeQUIEPR6snZFmpFpYvVsGnfLPNXbE=
|
||||
github.com/igm/sockjs-go/v3 v3.0.2/go.mod h1:UqchsOjeagIBFHvd+RZpLaVRbCwGilEC08EDHsD1jYE=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf h1:FtEj8sfIcaaBfAKrE1Cwb61YDtYq9JxChK1c7AKce7s=
|
||||
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf/go.mod h1:yrqSXGoD/4EKfF26AOGzscPOgTTJcyAwM2rpixWT+t4=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
@ -218,8 +206,6 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C
|
||||
github.com/keegancsmith/rpc v1.3.0 h1:wGWOpjcNrZaY8GDYZJfvyxmlLljm3YQWF+p918DXtDk=
|
||||
github.com/keegancsmith/rpc v1.3.0/go.mod h1:6O2xnOGjPyvIPbvp0MdrOe5r6cu1GZ4JoTzpzDhWeo0=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk=
|
||||
github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
@ -228,8 +214,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w=
|
||||
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
|
||||
github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
|
||||
github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
|
||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||
@ -243,23 +229,18 @@ github.com/lestrrat-go/jwx/v2 v2.0.8/go.mod h1:zLxnyv9rTlEvOUHbc48FAfIL8iYu2hHvI
|
||||
github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4=
|
||||
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
|
||||
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
|
||||
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
|
||||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
|
||||
github.com/miekg/dns v1.1.53 h1:ZBkuHr5dxHtB1caEOlZTLPo7D3L3TWckgUUs/RHfDxw=
|
||||
github.com/miekg/dns v1.1.53/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4=
|
||||
github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
@ -270,9 +251,6 @@ github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HD
|
||||
github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
|
||||
github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o=
|
||||
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
|
||||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
@ -284,9 +262,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
@ -302,11 +279,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8=
|
||||
github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0=
|
||||
github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
|
||||
@ -319,8 +293,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
|
||||
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648 h1:t2UQmCmUoElIBBuVTqxqo8DcTJA/exQ/Q7XycfLqCZo=
|
||||
gitlab.com/wpetit/goweb v0.0.0-20231019192040-4c72331a7648/go.mod h1:WdxGjM3HJWgBkUa4TwaTXUqY2BnRKlNSyUIv1aF4jxk=
|
||||
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b h1:nkvOl8TCj/mErADnwFFynjxBtC+hHsrESw6rw56JGmg=
|
||||
gitlab.com/wpetit/goweb v0.0.0-20230419082146-a94d9ed7202b/go.mod h1:3sus4zjoUv1GB7eDLL60QaPkUnXJCWBpjvbe0jWifeY=
|
||||
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
|
||||
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
|
||||
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
@ -328,8 +302,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
|
||||
go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
|
||||
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
@ -338,9 +310,8 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A=
|
||||
golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
|
||||
golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM=
|
||||
golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@ -374,7 +345,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
|
||||
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/net v0.0.0-20161013035702-8b4af36cd21a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -413,10 +383,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx
|
||||
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
|
||||
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
|
||||
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
|
||||
golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@ -437,7 +405,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@ -479,17 +446,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
|
||||
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
|
||||
golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28=
|
||||
golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo=
|
||||
golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@ -499,10 +462,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
|
||||
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@ -551,7 +512,6 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f
|
||||
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
|
||||
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
@ -645,11 +605,8 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
|
||||
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM=
|
||||
google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
@ -679,9 +636,7 @@ modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
|
||||
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
|
||||
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
|
||||
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
|
||||
modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
|
||||
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
|
||||
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
|
||||
modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
|
||||
modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
|
||||
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
|
||||
@ -695,11 +650,9 @@ modernc.org/sqlite v1.20.4/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A
|
||||
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
|
||||
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
|
||||
modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
|
||||
modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE=
|
||||
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
|
||||
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
|
||||
modernc.org/z v1.7.0/go.mod h1:hVdgNMh8ggTuRG1rGU8x+xGRFfiQUIAw0ZqlPy8+HyQ=
|
||||
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
|
||||
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
|
||||
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
|
||||
|
||||
@ -4,7 +4,7 @@ ARG HTTP_PROXY=
|
||||
ARG HTTPS_PROXY=
|
||||
ARG http_proxy=
|
||||
ARG https_proxy=
|
||||
ARG GO_VERSION=1.21.5
|
||||
ARG GO_VERSION=1.20.2
|
||||
|
||||
# Install dev environment dependencies
|
||||
RUN export DEBIAN_FRONTEND=noninteractive &&\
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
/var/log/storage-server/storage-server.log {
|
||||
missingok
|
||||
sharedscripts
|
||||
compress
|
||||
rotate 7
|
||||
postrotate
|
||||
/etc/init.d/storage-server restart
|
||||
endscript
|
||||
}
|
||||
@ -1,9 +1,9 @@
|
||||
#!/sbin/openrc-run
|
||||
|
||||
command="/usr/bin/storage-server"
|
||||
command_args="run"
|
||||
command_args=run""
|
||||
supervisor=supervise-daemon
|
||||
output_log="/var/log/storage-server/storage-server.log"
|
||||
output_log="/var/log/storage-server.log"
|
||||
error_log="$output_log"
|
||||
|
||||
depend() {
|
||||
|
||||
@ -4,7 +4,6 @@ pkg/sdk/client/src/**/*.js
|
||||
pkg/sdk/client/src/**/*.ts
|
||||
misc/client-sdk-testsuite/dist/server/*.js
|
||||
modd.conf
|
||||
.env
|
||||
{
|
||||
prep: make build-sdk build-cli build-storage-server
|
||||
daemon: make run-app
|
||||
@ -17,5 +16,5 @@ misc/client-sdk-testsuite/src/**/*
|
||||
}
|
||||
|
||||
**/*.go {
|
||||
# prep: make GOTEST_ARGS="-short" test
|
||||
prep: make GOTEST_ARGS="-short" test
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
ModuleFactories []ServerModuleFactory
|
||||
ErrorHandler func(ctx context.Context, err error)
|
||||
}
|
||||
|
||||
type OptionFunc func(opts *Options)
|
||||
|
||||
func NewOptions(funcs ...OptionFunc) *Options {
|
||||
opts := &Options{
|
||||
ModuleFactories: make([]ServerModuleFactory, 0),
|
||||
ErrorHandler: func(ctx context.Context, err error) {
|
||||
logger.Error(ctx, err.Error(), logger.E(errors.WithStack(err)))
|
||||
},
|
||||
}
|
||||
|
||||
for _, fn := range funcs {
|
||||
fn(opts)
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
func WithModulesFactories(factories ...ServerModuleFactory) OptionFunc {
|
||||
return func(opts *Options) {
|
||||
opts.ModuleFactories = factories
|
||||
}
|
||||
}
|
||||
@ -46,11 +46,7 @@ func NewPromiseProxyFrom(rt *goja.Runtime) *PromiseProxy {
|
||||
return NewPromiseProxy(promise, resolve, reject)
|
||||
}
|
||||
|
||||
func isPromise(v any) (*goja.Promise, bool) {
|
||||
if v == nil {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
promise, ok := v.(*goja.Promise)
|
||||
func IsPromise(v goja.Value) (*goja.Promise, bool) {
|
||||
promise, ok := v.Export().(*goja.Promise)
|
||||
return promise, ok
|
||||
}
|
||||
|
||||
@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
@ -14,7 +13,7 @@ import (
|
||||
|
||||
var (
|
||||
ErrFuncDoesNotExist = errors.New("function does not exist")
|
||||
ErrUnknownError = errors.New("unknown error")
|
||||
ErUnknownError = errors.New("unknown error")
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
@ -23,7 +22,23 @@ type Server struct {
|
||||
modules []ServerModule
|
||||
}
|
||||
|
||||
func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...any) (any, error) {
|
||||
func (s *Server) Load(name string, src string) error {
|
||||
var err error
|
||||
|
||||
s.loop.RunOnLoop(func(rt *goja.Runtime) {
|
||||
_, err = rt.RunScript(name, src)
|
||||
if err != nil {
|
||||
err = errors.Wrap(err, "could not run js script")
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...interface{}) (goja.Value, error) {
|
||||
ctx = logger.With(ctx, logger.F("function", funcName), logger.F("args", args))
|
||||
|
||||
ret, err := s.Exec(ctx, funcName, args...)
|
||||
@ -34,23 +49,16 @@ func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...an
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...any) (any, error) {
|
||||
type result struct {
|
||||
value any
|
||||
func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...interface{}) (goja.Value, error) {
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
value goja.Value
|
||||
err error
|
||||
}
|
||||
)
|
||||
|
||||
done := make(chan result)
|
||||
|
||||
defer func() {
|
||||
// Drain done channel
|
||||
for range done {
|
||||
}
|
||||
}()
|
||||
wg.Add(1)
|
||||
|
||||
s.loop.RunOnLoop(func(rt *goja.Runtime) {
|
||||
defer close(done)
|
||||
|
||||
var callable goja.Callable
|
||||
switch typ := callableOrFuncname.(type) {
|
||||
case goja.Callable:
|
||||
@ -59,9 +67,7 @@ func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...any)
|
||||
case string:
|
||||
call, ok := goja.AssertFunction(rt.Get(typ))
|
||||
if !ok {
|
||||
done <- result{
|
||||
err: errors.WithStack(ErrFuncDoesNotExist),
|
||||
}
|
||||
err = errors.WithStack(ErrFuncDoesNotExist)
|
||||
|
||||
return
|
||||
}
|
||||
@ -69,27 +75,28 @@ func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...any)
|
||||
callable = call
|
||||
|
||||
default:
|
||||
done <- result{
|
||||
err: errors.Errorf("callableOrFuncname: expected callable or function name, got '%T'", callableOrFuncname),
|
||||
}
|
||||
err = errors.Errorf("callableOrFuncname: expected callable or function name, got '%T'", callableOrFuncname)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "executing callable")
|
||||
|
||||
defer wg.Done()
|
||||
|
||||
defer func() {
|
||||
recovered := recover()
|
||||
if recovered == nil {
|
||||
if recovered := recover(); recovered != nil {
|
||||
revoveredErr, ok := recovered.(error)
|
||||
if ok {
|
||||
logger.Error(ctx, "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
|
||||
|
||||
err = errors.WithStack(ErUnknownError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
recoveredErr, ok := recovered.(error)
|
||||
if !ok {
|
||||
panic(recovered)
|
||||
}
|
||||
|
||||
done <- result{
|
||||
err: recoveredErr,
|
||||
}
|
||||
}()
|
||||
|
||||
jsArgs := make([]goja.Value, 0, len(args))
|
||||
@ -97,50 +104,25 @@ func (s *Server) Exec(ctx context.Context, callableOrFuncname any, args ...any)
|
||||
jsArgs = append(jsArgs, rt.ToValue(a))
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "executing callable", logger.F("callable", callableOrFuncname))
|
||||
|
||||
start := time.Now()
|
||||
value, err := callable(nil, jsArgs...)
|
||||
value, err = callable(nil, jsArgs...)
|
||||
if err != nil {
|
||||
done <- result{
|
||||
err: errors.WithStack(err),
|
||||
err = errors.WithStack(err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
done <- result{
|
||||
value: value.Export(),
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "executed callable", logger.F("callable", callableOrFuncname), logger.F("duration", time.Since(start).String()))
|
||||
})
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if err := ctx.Err(); err != nil {
|
||||
wg.Wait()
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
|
||||
case result := <-done:
|
||||
if result.err != nil {
|
||||
return nil, errors.WithStack(result.err)
|
||||
return value, nil
|
||||
}
|
||||
|
||||
if promise, ok := isPromise(result.value); ok {
|
||||
return s.waitForPromise(promise), nil
|
||||
}
|
||||
|
||||
return result.value, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) waitForPromise(promise *goja.Promise) any {
|
||||
func (s *Server) WaitForPromise(promise *goja.Promise) goja.Value {
|
||||
var (
|
||||
wg sync.WaitGroup
|
||||
value any
|
||||
value goja.Value
|
||||
)
|
||||
|
||||
wg.Add(1)
|
||||
@ -160,7 +142,7 @@ func (s *Server) waitForPromise(promise *goja.Promise) any {
|
||||
return
|
||||
}
|
||||
|
||||
value = promise.Result().Export()
|
||||
value = promise.Result()
|
||||
|
||||
breakLoop = true
|
||||
})
|
||||
@ -180,40 +162,20 @@ func (s *Server) waitForPromise(promise *goja.Promise) any {
|
||||
return value
|
||||
}
|
||||
|
||||
func (s *Server) Start(ctx context.Context, name string, src string) error {
|
||||
func (s *Server) Start() error {
|
||||
s.loop.Start()
|
||||
|
||||
done := make(chan error)
|
||||
var err error
|
||||
|
||||
s.loop.RunOnLoop(func(rt *goja.Runtime) {
|
||||
defer close(done)
|
||||
|
||||
rt.SetFieldNameMapper(goja.TagFieldNameMapper("goja", true))
|
||||
rt.SetRandSource(createRandomSource())
|
||||
|
||||
if err := s.loadModules(ctx, rt); err != nil {
|
||||
if err = s.initModules(rt); err != nil {
|
||||
err = errors.WithStack(err)
|
||||
done <- err
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := rt.RunScript(name, src); err != nil {
|
||||
done <- errors.Wrap(err, "could not run js script")
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.initModules(ctx, rt); err != nil {
|
||||
err = errors.WithStack(err)
|
||||
done <- err
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
done <- nil
|
||||
})
|
||||
|
||||
if err := <-done; err != nil {
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@ -224,7 +186,7 @@ func (s *Server) Stop() {
|
||||
s.loop.Stop()
|
||||
}
|
||||
|
||||
func (s *Server) loadModules(ctx context.Context, rt *goja.Runtime) error {
|
||||
func (s *Server) initModules(rt *goja.Runtime) error {
|
||||
modules := make([]ServerModule, 0, len(s.factories))
|
||||
|
||||
for _, moduleFactory := range s.factories {
|
||||
@ -238,25 +200,21 @@ func (s *Server) loadModules(ctx context.Context, rt *goja.Runtime) error {
|
||||
modules = append(modules, mod)
|
||||
}
|
||||
|
||||
s.modules = modules
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) initModules(ctx context.Context, rt *goja.Runtime) error {
|
||||
for _, mod := range s.modules {
|
||||
for _, mod := range modules {
|
||||
initMod, ok := mod.(InitializableModule)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "initializing module", logger.F("module", initMod.Name()))
|
||||
logger.Debug(context.Background(), "initializing module", logger.F("module", initMod.Name()))
|
||||
|
||||
if err := initMod.OnInit(ctx, rt); err != nil {
|
||||
if err := initMod.OnInit(rt); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
s.modules = modules
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -1,8 +1,6 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
)
|
||||
|
||||
@ -15,5 +13,5 @@ type ServerModule interface {
|
||||
|
||||
type InitializableModule interface {
|
||||
ServerModule
|
||||
OnInit(ctx context.Context, rt *goja.Runtime) error
|
||||
OnInit(rt *goja.Runtime) error
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ package bundle
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@ -40,6 +40,8 @@ func (fs *FileSystem) Open(name string) (http.File, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logger.Error(ctx, "could not open bundle file", logger.E(err))
|
||||
|
||||
return nil, errors.Wrapf(err, "could not open bundle file '%s'", p)
|
||||
}
|
||||
defer readCloser.Close()
|
||||
@ -51,14 +53,16 @@ func (fs *FileSystem) Open(name string) (http.File, error) {
|
||||
if fileInfo.IsDir() {
|
||||
files, err := fs.bundle.Dir(p)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not read bundle directory", logger.E(err))
|
||||
|
||||
return nil, errors.Wrapf(err, "could not read bundle directory '%s'", p)
|
||||
}
|
||||
|
||||
file.files = files
|
||||
} else {
|
||||
data, err := io.ReadAll(readCloser)
|
||||
data, err := ioutil.ReadAll(readCloser)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not read bundle file", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not read bundle file", logger.E(err))
|
||||
|
||||
return nil, errors.Wrapf(err, "could not read bundle file '%s'", p)
|
||||
}
|
||||
|
||||
@ -13,7 +13,6 @@ type ArchiveExt string
|
||||
const (
|
||||
ExtZip ArchiveExt = "zip"
|
||||
ExtTarGz ArchiveExt = "tar.gz"
|
||||
ExtZim ArchiveExt = "zim"
|
||||
)
|
||||
|
||||
func FromPath(path string) (Bundle, error) {
|
||||
@ -57,14 +56,5 @@ func matchArchivePattern(archivePath string) (Bundle, error) {
|
||||
return NewZipBundle(archivePath), nil
|
||||
}
|
||||
|
||||
matches, err = filepath.Match(fmt.Sprintf("*.%s", ExtZim), base)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "could not match file archive '%s'", archivePath)
|
||||
}
|
||||
|
||||
if matches {
|
||||
return NewZimBundle(archivePath), nil
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrUnknownBundleArchiveExt)
|
||||
}
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
package zim
|
||||
|
||||
import "io"
|
||||
|
||||
type BlobReader interface {
|
||||
io.ReadCloser
|
||||
Size() (int64, error)
|
||||
}
|
||||
@ -1,163 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type CompressedBlobReader struct {
|
||||
reader *Reader
|
||||
decoderFactory BlobDecoderFactory
|
||||
|
||||
clusterStartOffset uint64
|
||||
clusterEndOffset uint64
|
||||
blobIndex uint32
|
||||
blobSize int
|
||||
readOffset uint64
|
||||
|
||||
loadCluster sync.Once
|
||||
loadClusterErr error
|
||||
|
||||
data []byte
|
||||
closed bool
|
||||
}
|
||||
|
||||
// Size implements BlobReader.
|
||||
func (r *CompressedBlobReader) Size() (int64, error) {
|
||||
if err := r.loadClusterData(); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return int64(len(r.data)), nil
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *CompressedBlobReader) Close() error {
|
||||
clear(r.data)
|
||||
r.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *CompressedBlobReader) Read(p []byte) (int, error) {
|
||||
if err := r.loadClusterData(); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
length := len(p)
|
||||
remaining := len(r.data) - int(r.readOffset)
|
||||
if length > remaining {
|
||||
length = remaining
|
||||
}
|
||||
|
||||
chunk := make([]byte, length)
|
||||
|
||||
copy(chunk, r.data[r.readOffset:int(r.readOffset)+length])
|
||||
copy(p, chunk)
|
||||
|
||||
if length == remaining {
|
||||
return length, io.EOF
|
||||
}
|
||||
|
||||
r.readOffset += uint64(length)
|
||||
|
||||
return length, nil
|
||||
}
|
||||
|
||||
func (r *CompressedBlobReader) loadClusterData() error {
|
||||
if r.closed {
|
||||
return errors.WithStack(os.ErrClosed)
|
||||
}
|
||||
|
||||
r.loadCluster.Do(func() {
|
||||
compressedData := make([]byte, r.clusterEndOffset-r.clusterStartOffset)
|
||||
if err := r.reader.readRange(int64(r.clusterStartOffset+1), compressedData); err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobBuffer := bytes.NewBuffer(compressedData)
|
||||
|
||||
decoder, err := r.decoderFactory(blobBuffer)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer decoder.Close()
|
||||
|
||||
uncompressedData, err := io.ReadAll(decoder)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
blobStart uint64
|
||||
blobEnd uint64
|
||||
)
|
||||
|
||||
if r.blobSize == 8 {
|
||||
blobStart64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobStart = blobStart64
|
||||
|
||||
blobEnd64, err := readUint64(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobEnd = blobEnd64
|
||||
} else {
|
||||
blobStart32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobStart = uint64(blobStart32)
|
||||
|
||||
blobEnd32, err := readUint32(uncompressedData[r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize):r.blobIndex*uint32(r.blobSize)+uint32(r.blobSize)+uint32(r.blobSize)], binary.LittleEndian)
|
||||
if err != nil {
|
||||
r.loadClusterErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd32)
|
||||
}
|
||||
|
||||
r.data = make([]byte, blobEnd-blobStart)
|
||||
copy(r.data, uncompressedData[blobStart:blobEnd])
|
||||
})
|
||||
if r.loadClusterErr != nil {
|
||||
return errors.WithStack(r.loadClusterErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type BlobDecoderFactory func(io.Reader) (io.ReadCloser, error)
|
||||
|
||||
func NewCompressedBlobReader(reader *Reader, decoderFactory BlobDecoderFactory, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return &CompressedBlobReader{
|
||||
reader: reader,
|
||||
decoderFactory: decoderFactory,
|
||||
clusterStartOffset: clusterStartOffset,
|
||||
clusterEndOffset: clusterEndOffset,
|
||||
blobIndex: blobIndex,
|
||||
blobSize: blobSize,
|
||||
readOffset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
var _ BlobReader = &UncompressedBlobReader{}
|
||||
@ -1,193 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type zimCompression int
|
||||
|
||||
const (
|
||||
zimCompressionNoneZeno zimCompression = 0
|
||||
zimCompressionNone zimCompression = 1
|
||||
zimCompressionNoneZLib zimCompression = 2
|
||||
zimCompressionNoneBZip2 zimCompression = 3
|
||||
zimCompressionNoneXZ zimCompression = 4
|
||||
zimCompressionNoneZStandard zimCompression = 5
|
||||
)
|
||||
|
||||
type ContentEntry struct {
|
||||
*BaseEntry
|
||||
mimeType string
|
||||
clusterIndex uint32
|
||||
blobIndex uint32
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Compression() (int, error) {
|
||||
clusterHeader, _, _, err := e.readClusterInfo()
|
||||
if err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return int((clusterHeader << 4) >> 4), nil
|
||||
}
|
||||
|
||||
func (e *ContentEntry) MimeType() string {
|
||||
return e.mimeType
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Reader() (BlobReader, error) {
|
||||
clusterHeader, clusterStartOffset, clusterEndOffset, err := e.readClusterInfo()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
compression := (clusterHeader << 4) >> 4
|
||||
extended := (clusterHeader<<3)>>7 == 1
|
||||
|
||||
blobSize := 4
|
||||
if extended {
|
||||
blobSize = 8
|
||||
}
|
||||
|
||||
switch compression {
|
||||
|
||||
// Uncompressed blobs
|
||||
case uint8(zimCompressionNoneZeno):
|
||||
fallthrough
|
||||
case uint8(zimCompressionNone):
|
||||
startPos := clusterStartOffset + 1
|
||||
blobOffset := uint64(e.blobIndex * uint32(blobSize))
|
||||
|
||||
data := make([]byte, 2*blobSize)
|
||||
if err := e.reader.readRange(int64(startPos+blobOffset), data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var (
|
||||
blobStart uint64
|
||||
blobEnd uint64
|
||||
)
|
||||
|
||||
if extended {
|
||||
blobStart64, err := readUint64(data[0:blobSize], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobStart = blobStart64
|
||||
|
||||
blobEnd64, err := readUint64(data[blobSize:blobSize*2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd64)
|
||||
} else {
|
||||
blobStart32, err := readUint32(data[0:blobSize], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobStart = uint64(blobStart32)
|
||||
|
||||
blobEnd32, err := readUint32(data[blobSize:blobSize*2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
blobEnd = uint64(blobEnd32)
|
||||
}
|
||||
|
||||
return NewUncompressedBlobReader(e.reader, startPos+blobStart, startPos+blobEnd, blobSize), nil
|
||||
|
||||
// Supported compression algorithms
|
||||
case uint8(zimCompressionNoneXZ):
|
||||
return NewXZBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
|
||||
|
||||
case uint8(zimCompressionNoneZStandard):
|
||||
return NewZStdBlobReader(e.reader, clusterStartOffset, clusterEndOffset, e.blobIndex, blobSize), nil
|
||||
|
||||
// Unsupported compression algorithms
|
||||
case uint8(zimCompressionNoneZLib):
|
||||
fallthrough
|
||||
case uint8(zimCompressionNoneBZip2):
|
||||
fallthrough
|
||||
default:
|
||||
return nil, errors.Wrapf(ErrCompressionAlgorithmNotSupported, "unexpected compression algorithm '%d'", compression)
|
||||
}
|
||||
}
|
||||
|
||||
func (e *ContentEntry) Redirect() (*ContentEntry, error) {
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (e *ContentEntry) readClusterInfo() (uint8, uint64, uint64, error) {
|
||||
startClusterOffset, clusterEndOffset, err := e.reader.getClusterOffsets(int(e.clusterIndex))
|
||||
if err != nil {
|
||||
return 0, 0, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
data := make([]byte, 1)
|
||||
if err := e.reader.readRange(int64(startClusterOffset), data); err != nil {
|
||||
return 0, 0, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
clusterHeader := uint8(data[0])
|
||||
|
||||
return clusterHeader, startClusterOffset, clusterEndOffset, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseContentEntry(offset int64, base *BaseEntry) (*ContentEntry, error) {
|
||||
entry := &ContentEntry{
|
||||
BaseEntry: base,
|
||||
}
|
||||
|
||||
data := make([]byte, 16)
|
||||
if err := r.readRange(offset, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if mimeTypeIndex >= uint16(len(r.mimeTypes)) {
|
||||
return nil, errors.Errorf("mime type index '%d' greater than mime types length '%d'", mimeTypeIndex, len(r.mimeTypes))
|
||||
}
|
||||
|
||||
entry.mimeType = r.mimeTypes[mimeTypeIndex]
|
||||
|
||||
entry.namespace = Namespace(data[3:4])
|
||||
|
||||
clusterIndex, err := readUint32(data[8:12], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.clusterIndex = clusterIndex
|
||||
|
||||
blobIndex, err := readUint32(data[12:16], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.blobIndex = blobIndex
|
||||
|
||||
strs, _, err := r.readStringsAt(offset+16, 2, 1024)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(strs) > 0 {
|
||||
entry.url = strs[0]
|
||||
}
|
||||
|
||||
if len(strs) > 1 {
|
||||
entry.title = strs[1]
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
@ -1,135 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type Entry interface {
|
||||
Redirect() (*ContentEntry, error)
|
||||
Namespace() Namespace
|
||||
URL() string
|
||||
FullURL() string
|
||||
Title() string
|
||||
}
|
||||
|
||||
type BaseEntry struct {
|
||||
mimeTypeIndex uint16
|
||||
namespace Namespace
|
||||
url string
|
||||
title string
|
||||
reader *Reader
|
||||
}
|
||||
|
||||
func (e *BaseEntry) Namespace() Namespace {
|
||||
return e.namespace
|
||||
}
|
||||
|
||||
func (e *BaseEntry) Title() string {
|
||||
if e.title == "" {
|
||||
return e.url
|
||||
}
|
||||
|
||||
return e.title
|
||||
}
|
||||
|
||||
func (e *BaseEntry) URL() string {
|
||||
return e.url
|
||||
}
|
||||
|
||||
func (e *BaseEntry) FullURL() string {
|
||||
return toFullURL(e.Namespace(), e.URL())
|
||||
}
|
||||
|
||||
func (r *Reader) parseBaseEntry(offset int64) (*BaseEntry, error) {
|
||||
entry := &BaseEntry{
|
||||
reader: r,
|
||||
}
|
||||
|
||||
data := make([]byte, 3)
|
||||
if err := r.readRange(offset, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
mimeTypeIndex, err := readUint16(data[0:2], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.mimeTypeIndex = mimeTypeIndex
|
||||
entry.namespace = Namespace(data[2])
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
type RedirectEntry struct {
|
||||
*BaseEntry
|
||||
redirectIndex uint32
|
||||
}
|
||||
|
||||
func (e *RedirectEntry) Redirect() (*ContentEntry, error) {
|
||||
if e.redirectIndex >= uint32(len(e.reader.urlIndex)) {
|
||||
return nil, errors.Wrapf(ErrInvalidIndex, "entry index '%d' out of bounds", e.redirectIndex)
|
||||
}
|
||||
|
||||
entryPtr := e.reader.urlIndex[e.redirectIndex]
|
||||
entry, err := e.reader.parseEntryAt(int64(entryPtr))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry, err = entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
contentEntry, ok := entry.(*ContentEntry)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(ErrInvalidRedirect)
|
||||
}
|
||||
|
||||
return contentEntry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseRedirectEntry(offset int64, base *BaseEntry) (*RedirectEntry, error) {
|
||||
entry := &RedirectEntry{
|
||||
BaseEntry: base,
|
||||
}
|
||||
|
||||
data := make([]byte, 4)
|
||||
if err := r.readRange(offset+8, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
redirectIndex, err := readUint32(data, binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
entry.redirectIndex = redirectIndex
|
||||
|
||||
strs, _, err := r.readStringsAt(offset+12, 2, 1024)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(strs) > 0 {
|
||||
entry.url = strs[0]
|
||||
}
|
||||
|
||||
if len(strs) > 1 {
|
||||
entry.title = strs[1]
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func toFullURL(ns Namespace, url string) string {
|
||||
if ns == "\x00" {
|
||||
return url
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s/%s", ns, url)
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package zim
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
type EntryIterator struct {
|
||||
index int
|
||||
entry Entry
|
||||
err error
|
||||
reader *Reader
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Next() bool {
|
||||
if it.err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
entryCount := it.reader.EntryCount()
|
||||
|
||||
if it.index >= int(entryCount-1) {
|
||||
return false
|
||||
}
|
||||
|
||||
entry, err := it.reader.EntryAt(it.index)
|
||||
if err != nil {
|
||||
it.err = errors.WithStack(err)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
it.entry = entry
|
||||
it.index++
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Index() int {
|
||||
return it.index - 1
|
||||
}
|
||||
|
||||
func (it *EntryIterator) Entry() Entry {
|
||||
return it.entry
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
package zim
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrInvalidIndex = errors.New("invalid index")
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrInvalidRedirect = errors.New("invalid redirect")
|
||||
ErrCompressionAlgorithmNotSupported = errors.New("compression algorithm not supported")
|
||||
)
|
||||
@ -1,66 +0,0 @@
|
||||
package zim
|
||||
|
||||
import "github.com/pkg/errors"
|
||||
|
||||
func (r *Reader) Favicon() (*ContentEntry, error) {
|
||||
illustration, err := r.getMetadataIllustration()
|
||||
if err != nil && !errors.Is(err, ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if illustration != nil {
|
||||
return illustration, nil
|
||||
}
|
||||
|
||||
namespaces := []Namespace{V5NamespaceLayout, V5NamespaceImageFile}
|
||||
urls := []string{"favicon", "favicon.png"}
|
||||
|
||||
for _, ns := range namespaces {
|
||||
for _, url := range urls {
|
||||
entry, err := r.EntryWithURL(ns, url)
|
||||
if err != nil && !errors.Is(err, ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
func (r *Reader) getMetadataIllustration() (*ContentEntry, error) {
|
||||
keys := []MetadataKey{MetadataIllustration96x96at2, MetadataIllustration48x48at1}
|
||||
|
||||
metadata, err := r.Metadata(keys...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for _, k := range keys {
|
||||
if _, exists := metadata[k]; exists {
|
||||
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(k))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return content, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type MetadataKey string
|
||||
|
||||
// See https://wiki.openzim.org/wiki/Metadata
|
||||
const (
|
||||
MetadataName MetadataKey = "Name"
|
||||
MetadataTitle MetadataKey = "Title"
|
||||
MetadataDescription MetadataKey = "Description"
|
||||
MetadataLongDescription MetadataKey = "LongDescription"
|
||||
MetadataCreator MetadataKey = "Creator"
|
||||
MetadataTags MetadataKey = "Tags"
|
||||
MetadataDate MetadataKey = "Date"
|
||||
MetadataPublisher MetadataKey = "Publisher"
|
||||
MetadataFlavour MetadataKey = "Flavour"
|
||||
MetadataSource MetadataKey = "Source"
|
||||
MetadataLanguage MetadataKey = "Language"
|
||||
MetadataIllustration48x48at1 MetadataKey = "Illustration_48x48@1"
|
||||
MetadataIllustration96x96at2 MetadataKey = "Illustration_96x96@2"
|
||||
)
|
||||
|
||||
var knownKeys = []MetadataKey{
|
||||
MetadataName,
|
||||
MetadataTitle,
|
||||
MetadataDescription,
|
||||
MetadataLongDescription,
|
||||
MetadataCreator,
|
||||
MetadataPublisher,
|
||||
MetadataLanguage,
|
||||
MetadataTags,
|
||||
MetadataDate,
|
||||
MetadataFlavour,
|
||||
MetadataSource,
|
||||
MetadataIllustration48x48at1,
|
||||
MetadataIllustration96x96at2,
|
||||
}
|
||||
|
||||
// Metadata returns a copy of the internal metadata map of the ZIM file.
|
||||
func (r *Reader) Metadata(keys ...MetadataKey) (map[MetadataKey]string, error) {
|
||||
if len(keys) == 0 {
|
||||
keys = knownKeys
|
||||
}
|
||||
|
||||
metadata := make(map[MetadataKey]string)
|
||||
|
||||
for _, key := range keys {
|
||||
entry, err := r.EntryWithURL(V5NamespaceMetadata, string(key))
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
continue
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader, err := content.Reader()
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
metadata[key] = string(data)
|
||||
}
|
||||
|
||||
return metadata, nil
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
package zim
|
||||
|
||||
type Namespace string
|
||||
|
||||
const (
|
||||
V6NamespaceContent Namespace = "C"
|
||||
V6NamespaceMetadata Namespace = "M"
|
||||
V6NamespaceWellKnown Namespace = "W"
|
||||
V6NamespaceSearch Namespace = "X"
|
||||
)
|
||||
|
||||
const (
|
||||
V5NamespaceLayout Namespace = "-"
|
||||
V5NamespaceArticle Namespace = "A"
|
||||
V5NamespaceArticleMetadata Namespace = "B"
|
||||
V5NamespaceImageFile Namespace = "I"
|
||||
V5NamespaceImageText Namespace = "J"
|
||||
V5NamespaceMetadata Namespace = "M"
|
||||
V5NamespaceCategoryText Namespace = "U"
|
||||
V5NamespaceCategoryArticleList Namespace = "V"
|
||||
V5NamespaceCategoryPerArticle Namespace = "W"
|
||||
V5NamespaceSearch Namespace = "X"
|
||||
)
|
||||
@ -1,30 +0,0 @@
|
||||
package zim
|
||||
|
||||
import "time"
|
||||
|
||||
type Options struct {
|
||||
URLCacheSize int
|
||||
URLCacheTTL time.Duration
|
||||
CacheSize int
|
||||
}
|
||||
|
||||
type OptionFunc func(opts *Options)
|
||||
|
||||
func NewOptions(funcs ...OptionFunc) *Options {
|
||||
funcs = append([]OptionFunc{
|
||||
WithCacheSize(2048),
|
||||
}, funcs...)
|
||||
|
||||
opts := &Options{}
|
||||
for _, fn := range funcs {
|
||||
fn(opts)
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
func WithCacheSize(size int) OptionFunc {
|
||||
return func(opts *Options) {
|
||||
opts.CacheSize = size
|
||||
}
|
||||
}
|
||||
@ -1,558 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
const zimFormatMagicNumber uint32 = 0x44D495A
|
||||
const nullByte = '\x00'
|
||||
const zimRedirect = 0xffff
|
||||
|
||||
type Reader struct {
|
||||
majorVersion uint16
|
||||
minorVersion uint16
|
||||
uuid string
|
||||
entryCount uint32
|
||||
clusterCount uint32
|
||||
urlPtrPos uint64
|
||||
titlePtrPos uint64
|
||||
clusterPtrPos uint64
|
||||
mimeListPos uint64
|
||||
mainPage uint32
|
||||
layoutPage uint32
|
||||
checksumPos uint64
|
||||
|
||||
mimeTypes []string
|
||||
urlIndex []uint64
|
||||
clusterIndex []uint64
|
||||
|
||||
cache *lru.Cache[string, Entry]
|
||||
urls map[string]int
|
||||
|
||||
rangeReader RangeReadCloser
|
||||
}
|
||||
|
||||
func (r *Reader) Version() (majorVersion, minorVersion uint16) {
|
||||
return r.majorVersion, r.minorVersion
|
||||
}
|
||||
|
||||
func (r *Reader) EntryCount() uint32 {
|
||||
return r.entryCount
|
||||
}
|
||||
|
||||
func (r *Reader) ClusterCount() uint32 {
|
||||
return r.clusterCount
|
||||
}
|
||||
|
||||
func (r *Reader) UUID() string {
|
||||
return r.uuid
|
||||
}
|
||||
|
||||
func (r *Reader) Close() error {
|
||||
if err := r.rangeReader.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) MainPage() (Entry, error) {
|
||||
if r.mainPage == 0xffffffff {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
entry, err := r.EntryAt(int(r.mainPage))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) Entries() *EntryIterator {
|
||||
return &EntryIterator{
|
||||
reader: r,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reader) EntryAt(idx int) (Entry, error) {
|
||||
if idx >= len(r.urlIndex) || idx < 0 {
|
||||
return nil, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", idx)
|
||||
}
|
||||
|
||||
entryPtr := r.urlIndex[idx]
|
||||
|
||||
entry, err := r.parseEntryAt(int64(entryPtr))
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.cacheEntry(entryPtr, entry)
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithFullURL(url string) (Entry, error) {
|
||||
urlNum, exists := r.urls[url]
|
||||
if !exists {
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
entry, err := r.EntryAt(urlNum)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithURL(ns Namespace, url string) (Entry, error) {
|
||||
fullURL := toFullURL(ns, url)
|
||||
|
||||
entry, err := r.EntryWithFullURL(fullURL)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) EntryWithTitle(ns Namespace, title string) (Entry, error) {
|
||||
entry, found := r.getEntryByTitleFromCache(ns, title)
|
||||
if found {
|
||||
logger.Debug(context.Background(), "found entry with title from cache", logger.F("entry", entry.FullURL()))
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
iterator := r.Entries()
|
||||
|
||||
for iterator.Next() {
|
||||
entry := iterator.Entry()
|
||||
|
||||
if entry.Title() == title && entry.Namespace() == ns {
|
||||
return entry, nil
|
||||
}
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil, errors.WithStack(ErrNotFound)
|
||||
}
|
||||
|
||||
func (r *Reader) getURLCacheKey(fullURL string) string {
|
||||
return "url:" + fullURL
|
||||
}
|
||||
|
||||
func (r *Reader) getTitleCacheKey(ns Namespace, title string) string {
|
||||
return fmt.Sprintf("title:%s/%s", ns, title)
|
||||
}
|
||||
|
||||
func (r *Reader) cacheEntry(offset uint64, entry Entry) {
|
||||
urlKey := r.getURLCacheKey(entry.FullURL())
|
||||
titleKey := r.getTitleCacheKey(entry.Namespace(), entry.Title())
|
||||
|
||||
_, urlFound := r.cache.Peek(urlKey)
|
||||
_, titleFound := r.cache.Peek(titleKey)
|
||||
|
||||
if urlFound && titleFound {
|
||||
return
|
||||
}
|
||||
|
||||
r.cache.Add(urlKey, entry)
|
||||
r.cache.Add(titleKey, entry)
|
||||
}
|
||||
|
||||
func (r *Reader) getEntryByTitleFromCache(namespace Namespace, title string) (Entry, bool) {
|
||||
key := r.getTitleCacheKey(namespace, title)
|
||||
return r.cache.Get(key)
|
||||
}
|
||||
|
||||
func (r *Reader) parse() error {
|
||||
if err := r.parseHeader(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseMimeTypes(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseURLIndex(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := r.parseClusterIndex(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseHeader() error {
|
||||
header := make([]byte, 80)
|
||||
if err := r.readRange(0, header); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
magicNumber, err := readUint32(header[0:4], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if magicNumber != zimFormatMagicNumber {
|
||||
return errors.Errorf("invalid zim magic number '%d'", magicNumber)
|
||||
}
|
||||
|
||||
majorVersion, err := readUint16(header[4:6], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.majorVersion = majorVersion
|
||||
|
||||
minorVersion, err := readUint16(header[6:8], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.minorVersion = minorVersion
|
||||
|
||||
if err := r.parseUUID(header[8:16]); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
entryCount, err := readUint32(header[24:28], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.entryCount = entryCount
|
||||
|
||||
clusterCount, err := readUint32(header[28:32], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterCount = clusterCount
|
||||
|
||||
urlPtrPos, err := readUint64(header[32:40], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.urlPtrPos = urlPtrPos
|
||||
|
||||
titlePtrPos, err := readUint64(header[40:48], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.titlePtrPos = titlePtrPos
|
||||
|
||||
clusterPtrPos, err := readUint64(header[48:56], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterPtrPos = clusterPtrPos
|
||||
|
||||
mimeListPos, err := readUint64(header[56:64], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.mimeListPos = mimeListPos
|
||||
|
||||
mainPage, err := readUint32(header[64:68], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.mainPage = mainPage
|
||||
|
||||
layoutPage, err := readUint32(header[68:72], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.layoutPage = layoutPage
|
||||
|
||||
checksumPos, err := readUint64(header[72:80], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.checksumPos = checksumPos
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseUUID(data []byte) error {
|
||||
parts := make([]string, 0, 5)
|
||||
|
||||
val32, err := readUint32(data[0:4], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%08x", val32))
|
||||
|
||||
val16, err := readUint16(data[4:6], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val16, err = readUint16(data[6:8], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val16, err = readUint16(data[8:10], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%04x", val16))
|
||||
|
||||
val32, err = readUint32(data[10:14], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
val16, err = readUint16(data[14:16], binary.BigEndian)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
parts = append(parts, fmt.Sprintf("%x%x", val32, val16))
|
||||
|
||||
r.uuid = strings.Join(parts, "-")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseMimeTypes() error {
|
||||
mimeTypes := make([]string, 0)
|
||||
offset := int64(r.mimeListPos)
|
||||
read := int64(0)
|
||||
var err error
|
||||
var found []string
|
||||
for {
|
||||
found, read, err = r.readStringsAt(offset+read, 64, 1024)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if len(found) == 0 || found[0] == "" {
|
||||
break
|
||||
}
|
||||
|
||||
mimeTypes = append(mimeTypes, found...)
|
||||
}
|
||||
|
||||
r.mimeTypes = mimeTypes
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseURLIndex() error {
|
||||
urlIndex, err := r.parsePointerIndex(int64(r.urlPtrPos), int64(r.entryCount))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.urlIndex = urlIndex
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseClusterIndex() error {
|
||||
clusterIndex, err := r.parsePointerIndex(int64(r.clusterPtrPos), int64(r.clusterCount+1))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
r.clusterIndex = clusterIndex
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) parseEntryAt(offset int64) (Entry, error) {
|
||||
base, err := r.parseBaseEntry(offset)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var entry Entry
|
||||
|
||||
if base.mimeTypeIndex == zimRedirect {
|
||||
entry, err = r.parseRedirectEntry(offset, base)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
} else {
|
||||
entry, err = r.parseContentEntry(offset, base)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (r *Reader) parsePointerIndex(startAddr int64, count int64) ([]uint64, error) {
|
||||
index := make([]uint64, count)
|
||||
|
||||
data := make([]byte, count*8)
|
||||
if err := r.readRange(startAddr, data); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for i := int64(0); i < count; i++ {
|
||||
offset := i * 8
|
||||
ptr, err := readUint64(data[offset:offset+8], binary.LittleEndian)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
index[i] = ptr
|
||||
}
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (r *Reader) getClusterOffsets(clusterNum int) (uint64, uint64, error) {
|
||||
if clusterNum > len(r.clusterIndex)-1 || clusterNum < 0 {
|
||||
return 0, 0, errors.Wrapf(ErrInvalidIndex, "index '%d' out of bounds", clusterNum)
|
||||
}
|
||||
|
||||
return r.clusterIndex[clusterNum], r.clusterIndex[clusterNum+1] - 1, nil
|
||||
}
|
||||
|
||||
func (r *Reader) preload() error {
|
||||
r.urls = make(map[string]int, r.entryCount)
|
||||
|
||||
iterator := r.Entries()
|
||||
for iterator.Next() {
|
||||
entry := iterator.Entry()
|
||||
r.urls[entry.FullURL()] = iterator.Index()
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readRange(offset int64, v []byte) error {
|
||||
read, err := r.rangeReader.ReadAt(v, offset)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if read != len(v) {
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Reader) readStringsAt(offset int64, count int, bufferSize int) ([]string, int64, error) {
|
||||
var sb strings.Builder
|
||||
read := int64(0)
|
||||
|
||||
values := make([]string, 0, count)
|
||||
wasNullByte := false
|
||||
|
||||
for {
|
||||
data := make([]byte, bufferSize)
|
||||
err := r.readRange(offset+read, data)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, read, errors.WithStack(err)
|
||||
}
|
||||
|
||||
for idx := 0; idx < len(data); idx++ {
|
||||
d := data[idx]
|
||||
if err := sb.WriteByte(d); err != nil {
|
||||
return nil, read, errors.WithStack(err)
|
||||
}
|
||||
|
||||
read++
|
||||
|
||||
if d == nullByte {
|
||||
if wasNullByte {
|
||||
return values, read, nil
|
||||
}
|
||||
|
||||
wasNullByte = true
|
||||
|
||||
str := strings.TrimRight(sb.String(), "\x00")
|
||||
values = append(values, str)
|
||||
|
||||
if len(values) == count || errors.Is(err, io.EOF) {
|
||||
return values, read, nil
|
||||
}
|
||||
|
||||
sb.Reset()
|
||||
} else {
|
||||
wasNullByte = false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type RangeReadCloser interface {
|
||||
io.Closer
|
||||
ReadAt(data []byte, offset int64) (n int, err error)
|
||||
}
|
||||
|
||||
func NewReader(rangeReader RangeReadCloser, funcs ...OptionFunc) (*Reader, error) {
|
||||
opts := NewOptions(funcs...)
|
||||
|
||||
cache, err := lru.New[string, Entry](opts.CacheSize)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader := &Reader{
|
||||
rangeReader: rangeReader,
|
||||
cache: cache,
|
||||
}
|
||||
|
||||
if err := reader.parse(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := reader.preload(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
func Open(path string, funcs ...OptionFunc) (*Reader, error) {
|
||||
file, err := os.OpenFile(path, os.O_RDONLY, os.ModePerm)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
reader, err := NewReader(file, funcs...)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type readerTestCase struct {
|
||||
UUID string `json:"uuid"`
|
||||
EntryCount uint32 `json:"entryCount"`
|
||||
Entries []struct {
|
||||
Namespace Namespace `json:"namespace"`
|
||||
URL string `json:"url"`
|
||||
Size int64 `json:"size"`
|
||||
Compression int `json:"compression"`
|
||||
MimeType string `json:"mimeType"`
|
||||
Title string `json:"title"`
|
||||
} `json:"entries"`
|
||||
}
|
||||
|
||||
func TestReader(t *testing.T) {
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(logger.LevelDebug)
|
||||
logger.SetFormat(logger.FormatHuman)
|
||||
}
|
||||
|
||||
files, err := filepath.Glob("testdata/*.zim")
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
for _, zf := range files {
|
||||
testName := filepath.Base(zf)
|
||||
testCase, err := loadZimFileTestCase(zf)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
reader, err := Open(zf)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
}()
|
||||
|
||||
if e, g := testCase.UUID, reader.UUID(); e != g {
|
||||
t.Errorf("reader.UUID(): expected '%s', got '%s'", e, g)
|
||||
}
|
||||
|
||||
if e, g := testCase.EntryCount, reader.EntryCount(); e != g {
|
||||
t.Errorf("reader.EntryCount(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
if testCase.Entries == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, entryTestCase := range testCase.Entries {
|
||||
testName := fmt.Sprintf("Entry/%s/%s", entryTestCase.Namespace, entryTestCase.URL)
|
||||
t.Run(testName, func(t *testing.T) {
|
||||
entry, err := reader.EntryWithURL(entryTestCase.Namespace, entryTestCase.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.MimeType, content.MimeType(); e != g {
|
||||
t.Errorf("content.MimeType(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Title, content.Title(); e != g {
|
||||
t.Errorf("content.Title(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
compression, err := content.Compression()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Compression, compression; e != g {
|
||||
t.Errorf("content.Compression(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
contentReader, err := content.Reader()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
size, err := contentReader.Size()
|
||||
if err != nil {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
if e, g := entryTestCase.Size, size; e != g {
|
||||
t.Errorf("content.Size(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadZimFileTestCase(zimFile string) (*readerTestCase, error) {
|
||||
testCaseFile, _ := strings.CutSuffix(zimFile, ".zim")
|
||||
|
||||
data, err := os.ReadFile(testCaseFile + ".json")
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
testCase := &readerTestCase{}
|
||||
if err := json.Unmarshal(data, testCase); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return testCase, nil
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
{
|
||||
"uuid": "8d141c3b-115d-bf73-294a-ee3c2e6b97b0",
|
||||
"entryCount": 6223,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "users_page=9",
|
||||
"compression": 5,
|
||||
"size": 58646,
|
||||
"mimeType": "text/html",
|
||||
"title": "users_page=9"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
22
pkg/bundle/zim/testdata/cadoles.json
vendored
22
pkg/bundle/zim/testdata/cadoles.json
vendored
@ -1,22 +0,0 @@
|
||||
{
|
||||
"uuid": "cf81f094-d802-c790-b854-c74ad9701ddb",
|
||||
"entryCount": 271,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "blog/202206-ShowroomInnovation.jpg",
|
||||
"compression": 1,
|
||||
"size": 260260,
|
||||
"mimeType": "image/jpeg",
|
||||
"title": "blog/202206-ShowroomInnovation.jpg"
|
||||
},
|
||||
{
|
||||
"namespace": "C",
|
||||
"url": "team/index.html",
|
||||
"compression": 5,
|
||||
"size": 93185,
|
||||
"mimeType": "text/html",
|
||||
"title": "Cadoles - Notre équipe"
|
||||
}
|
||||
]
|
||||
}
|
||||
BIN
pkg/bundle/zim/testdata/cadoles.zim
vendored
BIN
pkg/bundle/zim/testdata/cadoles.zim
vendored
Binary file not shown.
@ -1,14 +0,0 @@
|
||||
{
|
||||
"uuid": "ad4f406c-2021-2db8-c729-297568bbe376",
|
||||
"entryCount": 330,
|
||||
"entries": [
|
||||
{
|
||||
"namespace": "M",
|
||||
"url": "Illustration_48x48@1",
|
||||
"compression": 5,
|
||||
"size": 5365,
|
||||
"mimeType": "text/plain",
|
||||
"title": "Illustration_48x48@1"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@ -1,86 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type UncompressedBlobReader struct {
|
||||
reader *Reader
|
||||
blobStartOffset uint64
|
||||
blobEndOffset uint64
|
||||
blobSize int
|
||||
readOffset int
|
||||
|
||||
blobData []byte
|
||||
loadBlobOnce sync.Once
|
||||
loadBlobErr error
|
||||
}
|
||||
|
||||
// Size implements BlobReader.
|
||||
func (r *UncompressedBlobReader) Size() (int64, error) {
|
||||
return int64(r.blobEndOffset - r.blobStartOffset), nil
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *UncompressedBlobReader) Close() error {
|
||||
clear(r.blobData)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *UncompressedBlobReader) Read(p []byte) (n int, err error) {
|
||||
blobData, err := r.loadBlob()
|
||||
if err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
chunkLength := len(p)
|
||||
remaining := int(len(blobData) - r.readOffset)
|
||||
if chunkLength > remaining {
|
||||
chunkLength = remaining
|
||||
}
|
||||
|
||||
chunk := blobData[r.readOffset : r.readOffset+chunkLength]
|
||||
r.readOffset += chunkLength
|
||||
|
||||
copy(p, chunk)
|
||||
|
||||
if chunkLength == remaining {
|
||||
return chunkLength, io.EOF
|
||||
}
|
||||
|
||||
return chunkLength, nil
|
||||
}
|
||||
|
||||
func (r *UncompressedBlobReader) loadBlob() ([]byte, error) {
|
||||
r.loadBlobOnce.Do(func() {
|
||||
data := make([]byte, r.blobEndOffset-r.blobStartOffset)
|
||||
err := r.reader.readRange(int64(r.blobStartOffset), data)
|
||||
if err != nil {
|
||||
r.loadBlobErr = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
|
||||
r.blobData = data
|
||||
})
|
||||
if r.loadBlobErr != nil {
|
||||
return nil, errors.WithStack(r.loadBlobErr)
|
||||
}
|
||||
|
||||
return r.blobData, nil
|
||||
}
|
||||
|
||||
func NewUncompressedBlobReader(reader *Reader, blobStartOffset, blobEndOffset uint64, blobSize int) *UncompressedBlobReader {
|
||||
return &UncompressedBlobReader{
|
||||
reader: reader,
|
||||
blobStartOffset: blobStartOffset,
|
||||
blobEndOffset: blobEndOffset,
|
||||
blobSize: blobSize,
|
||||
readOffset: 0,
|
||||
}
|
||||
}
|
||||
|
||||
var _ BlobReader = &UncompressedBlobReader{}
|
||||
@ -1,52 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// read a little endian uint64
|
||||
func readUint64(b []byte, order binary.ByteOrder) (uint64, error) {
|
||||
var v uint64
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint32
|
||||
func readUint32(b []byte, order binary.ByteOrder) (uint32, error) {
|
||||
var v uint32
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint16
|
||||
func readUint16(b []byte, order binary.ByteOrder) (uint16, error) {
|
||||
var v uint16
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// read a little endian uint8
|
||||
func readUint8(b []byte, order binary.ByteOrder) (uint8, error) {
|
||||
var v uint8
|
||||
buf := bytes.NewBuffer(b)
|
||||
if err := binary.Read(buf, order, &v); err != nil {
|
||||
return 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return v, nil
|
||||
}
|
||||
@ -1,42 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/ulikunitz/xz"
|
||||
)
|
||||
|
||||
type XZBlobReader struct {
|
||||
decoder *xz.Reader
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *XZBlobReader) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *XZBlobReader) Read(p []byte) (n int, err error) {
|
||||
return r.decoder.Read(p)
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &XZBlobReader{}
|
||||
|
||||
func NewXZBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return NewCompressedBlobReader(
|
||||
reader,
|
||||
func(r io.Reader) (io.ReadCloser, error) {
|
||||
decoder, err := xz.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &XZBlobReader{decoder}, nil
|
||||
},
|
||||
clusterStartOffset,
|
||||
clusterEndOffset,
|
||||
blobIndex,
|
||||
blobSize,
|
||||
)
|
||||
}
|
||||
@ -1,43 +0,0 @@
|
||||
package zim
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type ZstdBlobReader struct {
|
||||
decoder *zstd.Decoder
|
||||
}
|
||||
|
||||
// Close implements io.ReadCloser.
|
||||
func (r *ZstdBlobReader) Close() error {
|
||||
r.decoder.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements io.ReadCloser.
|
||||
func (r *ZstdBlobReader) Read(p []byte) (n int, err error) {
|
||||
return r.decoder.Read(p)
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &ZstdBlobReader{}
|
||||
|
||||
func NewZStdBlobReader(reader *Reader, clusterStartOffset, clusterEndOffset uint64, blobIndex uint32, blobSize int) *CompressedBlobReader {
|
||||
return NewCompressedBlobReader(
|
||||
reader,
|
||||
func(r io.Reader) (io.ReadCloser, error) {
|
||||
decoder, err := zstd.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &ZstdBlobReader{decoder}, nil
|
||||
},
|
||||
clusterStartOffset,
|
||||
clusterEndOffset,
|
||||
blobIndex,
|
||||
blobSize,
|
||||
)
|
||||
}
|
||||
@ -1,483 +0,0 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bundle/zim"
|
||||
lru "github.com/hashicorp/golang-lru/v2"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type ZimBundle struct {
|
||||
archivePath string
|
||||
|
||||
initOnce sync.Once
|
||||
initErr error
|
||||
|
||||
reader *zim.Reader
|
||||
urlNamespaceCache *lru.Cache[string, zim.Namespace]
|
||||
}
|
||||
|
||||
func (b *ZimBundle) File(filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
ctx := logger.With(
|
||||
context.Background(),
|
||||
logger.F("filename", filename),
|
||||
)
|
||||
|
||||
logger.Debug(ctx, "opening file")
|
||||
|
||||
switch filename {
|
||||
case "manifest.yml":
|
||||
return b.renderFakeManifest(ctx)
|
||||
case "server/main.js":
|
||||
return b.renderFakeServerMain(ctx)
|
||||
case "public":
|
||||
return b.renderDirectory(ctx, filename)
|
||||
case "public/index.html":
|
||||
return b.renderMainPage(ctx, filename)
|
||||
|
||||
default:
|
||||
return b.renderURL(ctx, filename)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *ZimBundle) Dir(dirname string) ([]os.FileInfo, error) {
|
||||
files := make([]os.FileInfo, 0)
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderFakeManifest(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
metadata, err := b.reader.Metadata()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
manifest := map[string]any{}
|
||||
|
||||
manifest["version"] = "0.0.0"
|
||||
|
||||
if name, exists := metadata[zim.MetadataName]; exists {
|
||||
replacer := strings.NewReplacer(
|
||||
"_", "",
|
||||
" ", "",
|
||||
)
|
||||
|
||||
manifest["id"] = strings.ToLower(replacer.Replace(name)) + ".zim.edge.app"
|
||||
} else {
|
||||
manifest["id"] = b.reader.UUID() + ".zim.edge.app"
|
||||
}
|
||||
|
||||
if title, exists := metadata[zim.MetadataTitle]; exists {
|
||||
manifest["title"] = title
|
||||
} else {
|
||||
manifest["title"] = "Unknown"
|
||||
}
|
||||
|
||||
if description, exists := metadata[zim.MetadataDescription]; exists {
|
||||
manifest["description"] = description
|
||||
}
|
||||
|
||||
favicon, err := b.reader.Favicon()
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if favicon != nil {
|
||||
manifestMeta, exists := manifest["metadata"].(map[string]any)
|
||||
if !exists {
|
||||
manifestMeta = make(map[string]any)
|
||||
manifest["metadata"] = manifestMeta
|
||||
}
|
||||
|
||||
paths, exists := manifestMeta["paths"].(map[string]any)
|
||||
if !exists {
|
||||
paths = make(map[string]any)
|
||||
manifestMeta["paths"] = paths
|
||||
}
|
||||
|
||||
paths["icon"] = "/" + favicon.FullURL()
|
||||
}
|
||||
|
||||
data, err := yaml.Marshal(manifest)
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
stat := &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: "manifest.yml",
|
||||
size: int64(len(data)),
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(data)
|
||||
file := io.NopCloser(buf)
|
||||
|
||||
return file, stat, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderFakeServerMain(ctx context.Context) (io.ReadCloser, os.FileInfo, error) {
|
||||
stat := &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: "server/main.js",
|
||||
size: 0,
|
||||
}
|
||||
|
||||
buf := bytes.NewBuffer(nil)
|
||||
file := io.NopCloser(buf)
|
||||
|
||||
return file, stat, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderURL(ctx context.Context, url string) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
url = strings.TrimPrefix(url, "public/")
|
||||
|
||||
entry, err := b.searchEntryFromURL(ctx, url)
|
||||
if err != nil {
|
||||
if errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, nil, os.ErrNotExist
|
||||
}
|
||||
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
logger.Debug(
|
||||
ctx, "found zim entry",
|
||||
logger.F("webURL", url),
|
||||
logger.F("zimFullURL", entry.FullURL()),
|
||||
)
|
||||
|
||||
content, err := entry.Redirect()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
contentReader, err := content.Reader()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
size, err := contentReader.Size()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
filename := filepath.Base(url)
|
||||
|
||||
mimeType := content.MimeType()
|
||||
if mimeType != "text/html" {
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: size,
|
||||
},
|
||||
reader: contentReader,
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
// Read HTML file and inject Edge scripts
|
||||
|
||||
data, err := io.ReadAll(contentReader)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
injected, err := b.injectEdgeScriptTag(data)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not inject edge script", logger.E(errors.WithStack(err)))
|
||||
} else {
|
||||
data = injected
|
||||
}
|
||||
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: false,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: size,
|
||||
},
|
||||
reader: io.NopCloser(bytes.NewBuffer(data)),
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) searchEntryFromURL(ctx context.Context, url string) (zim.Entry, error) {
|
||||
ctx = logger.With(ctx, logger.F("webURL", url))
|
||||
|
||||
logger.Debug(ctx, "searching entry namespace in local cache")
|
||||
|
||||
entry, err := b.reader.EntryWithFullURL(url)
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry != nil {
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
contentNamespaces := []zim.Namespace{
|
||||
zim.V6NamespaceContent,
|
||||
zim.V6NamespaceMetadata,
|
||||
zim.V5NamespaceLayout,
|
||||
zim.V5NamespaceArticle,
|
||||
zim.V5NamespaceImageFile,
|
||||
zim.V5NamespaceMetadata,
|
||||
}
|
||||
|
||||
logger.Debug(
|
||||
ctx, "make educated guesses about potential url namespace",
|
||||
logger.F("zimNamespaces", contentNamespaces),
|
||||
)
|
||||
|
||||
for _, ns := range contentNamespaces {
|
||||
logger.Debug(
|
||||
ctx, "trying to access entry directly",
|
||||
logger.F("zimNamespace", ns),
|
||||
logger.F("zimURL", url),
|
||||
)
|
||||
|
||||
entry, err := b.reader.EntryWithURL(ns, url)
|
||||
if err != nil && !errors.Is(err, zim.ErrNotFound) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry != nil {
|
||||
b.urlNamespaceCache.Add(url, entry.Namespace())
|
||||
return entry, nil
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "doing full entries scan")
|
||||
|
||||
iterator := b.reader.Entries()
|
||||
for iterator.Next() {
|
||||
current := iterator.Entry()
|
||||
|
||||
if current.FullURL() != url && current.URL() != url {
|
||||
continue
|
||||
}
|
||||
|
||||
entry = current
|
||||
b.urlNamespaceCache.Add(url, entry.Namespace())
|
||||
break
|
||||
}
|
||||
if err := iterator.Err(); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
return nil, errors.WithStack(zim.ErrNotFound)
|
||||
}
|
||||
|
||||
return entry, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderDirectory(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
zimFile := &zimFile{
|
||||
fileInfo: &zimFileInfo{
|
||||
isDir: true,
|
||||
modTime: time.Time{},
|
||||
mode: 0,
|
||||
name: filename,
|
||||
size: 0,
|
||||
},
|
||||
reader: io.NopCloser(bytes.NewBuffer(nil)),
|
||||
}
|
||||
|
||||
return zimFile, zimFile.fileInfo, nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) renderMainPage(ctx context.Context, filename string) (io.ReadCloser, os.FileInfo, error) {
|
||||
if err := b.init(); err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
main, err := b.reader.MainPage()
|
||||
if err != nil {
|
||||
return nil, nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return b.renderURL(ctx, main.FullURL())
|
||||
}
|
||||
|
||||
func (b *ZimBundle) injectEdgeScriptTag(data []byte) ([]byte, error) {
|
||||
buff := bytes.NewBuffer(data)
|
||||
doc, err := html.Parse(buff)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var f func(*html.Node) bool
|
||||
f = func(n *html.Node) bool {
|
||||
if n.Type == html.ElementNode && n.Data == "head" {
|
||||
script := &html.Node{
|
||||
Type: html.ElementNode,
|
||||
Data: "script",
|
||||
Attr: []html.Attribute{
|
||||
{
|
||||
Key: "src",
|
||||
Val: "/edge/sdk/client.js",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
n.AppendChild(script)
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if keepWalking := f(c); !keepWalking {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
f(doc)
|
||||
|
||||
buff.Reset()
|
||||
|
||||
if err := html.Render(buff, doc); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return buff.Bytes(), nil
|
||||
}
|
||||
|
||||
func (b *ZimBundle) init() error {
|
||||
b.initOnce.Do(func() {
|
||||
reader, err := zim.Open(b.archivePath)
|
||||
if err != nil {
|
||||
b.initErr = errors.Wrapf(err, "could not open '%v'", b.archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
b.reader = reader
|
||||
|
||||
cache, err := lru.New[string, zim.Namespace](128)
|
||||
if err != nil {
|
||||
b.initErr = errors.Wrap(err, "could not initialize cache")
|
||||
return
|
||||
}
|
||||
|
||||
b.urlNamespaceCache = cache
|
||||
})
|
||||
if b.initErr != nil {
|
||||
return errors.WithStack(b.initErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewZimBundle(archivePath string) *ZimBundle {
|
||||
return &ZimBundle{
|
||||
archivePath: archivePath,
|
||||
}
|
||||
}
|
||||
|
||||
type zimFile struct {
|
||||
fileInfo *zimFileInfo
|
||||
reader io.ReadCloser
|
||||
}
|
||||
|
||||
// Close implements fs.File.
|
||||
func (f *zimFile) Close() error {
|
||||
if err := f.reader.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read implements fs.File.
|
||||
func (f *zimFile) Read(d []byte) (int, error) {
|
||||
n, err := f.reader.Read(d)
|
||||
if err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
return n, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Stat implements fs.File.
|
||||
func (f *zimFile) Stat() (fs.FileInfo, error) {
|
||||
return f.fileInfo, nil
|
||||
}
|
||||
|
||||
var _ fs.File = &zimFile{}
|
||||
|
||||
type zimFileInfo struct {
|
||||
isDir bool
|
||||
modTime time.Time
|
||||
mode fs.FileMode
|
||||
name string
|
||||
size int64
|
||||
}
|
||||
|
||||
// IsDir implements fs.FileInfo.
|
||||
func (i *zimFileInfo) IsDir() bool {
|
||||
return i.isDir
|
||||
}
|
||||
|
||||
// ModTime implements fs.FileInfo.
|
||||
func (i *zimFileInfo) ModTime() time.Time {
|
||||
return i.modTime
|
||||
}
|
||||
|
||||
// Mode implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Mode() fs.FileMode {
|
||||
return i.mode
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Name() string {
|
||||
return i.name
|
||||
}
|
||||
|
||||
// Size implements fs.FileInfo.
|
||||
func (i *zimFileInfo) Size() int64 {
|
||||
return i.size
|
||||
}
|
||||
|
||||
// Sys implements fs.FileInfo.
|
||||
func (*zimFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ fs.FileInfo = &zimFileInfo{}
|
||||
@ -3,11 +3,11 @@ package bus
|
||||
import "context"
|
||||
|
||||
type Bus interface {
|
||||
Subscribe(ctx context.Context, addr Address) (<-chan Envelope, error)
|
||||
Unsubscribe(addr Address, ch <-chan Envelope)
|
||||
Publish(env Envelope) error
|
||||
Request(ctx context.Context, env Envelope) (Envelope, error)
|
||||
Reply(ctx context.Context, addr Address, h RequestHandler) chan error
|
||||
Subscribe(ctx context.Context, ns MessageNamespace) (<-chan Message, error)
|
||||
Unsubscribe(ctx context.Context, ns MessageNamespace, ch <-chan Message)
|
||||
Publish(ctx context.Context, msg Message) error
|
||||
Request(ctx context.Context, msg Message) (Message, error)
|
||||
Reply(ctx context.Context, ns MessageNamespace, h RequestHandler) error
|
||||
}
|
||||
|
||||
type RequestHandler func(env Envelope) (any, error)
|
||||
type RequestHandler func(msg Message) (Message, error)
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
package bus
|
||||
|
||||
type Address string
|
||||
|
||||
type Envelope interface {
|
||||
Message() any
|
||||
Address() Address
|
||||
}
|
||||
|
||||
type BaseEnvelope struct {
|
||||
msg any
|
||||
addr Address
|
||||
}
|
||||
|
||||
// Address implements Envelope.
|
||||
func (e *BaseEnvelope) Address() Address {
|
||||
return e.addr
|
||||
}
|
||||
|
||||
// Message implements Envelope.
|
||||
func (e *BaseEnvelope) Message() any {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
func NewEnvelope(addr Address, msg any) *BaseEnvelope {
|
||||
return &BaseEnvelope{
|
||||
addr: addr,
|
||||
msg: msg,
|
||||
}
|
||||
}
|
||||
|
||||
var _ Envelope = &BaseEnvelope{}
|
||||
@ -15,13 +15,13 @@ type Bus struct {
|
||||
nextRequestID uint64
|
||||
}
|
||||
|
||||
func (b *Bus) Subscribe(ctx context.Context, address bus.Address) (<-chan bus.Envelope, error) {
|
||||
func (b *Bus) Subscribe(ctx context.Context, ns bus.MessageNamespace) (<-chan bus.Message, error) {
|
||||
logger.Debug(
|
||||
ctx, "subscribing",
|
||||
logger.F("address", address),
|
||||
ctx, "subscribing to messages",
|
||||
logger.F("messageNamespace", ns),
|
||||
)
|
||||
|
||||
dispatchers := b.getDispatchers(address)
|
||||
dispatchers := b.getDispatchers(ns)
|
||||
disp := newEventDispatcher(b.opt.BufferSize)
|
||||
|
||||
go disp.Run(ctx)
|
||||
@ -31,41 +31,50 @@ func (b *Bus) Subscribe(ctx context.Context, address bus.Address) (<-chan bus.En
|
||||
return disp.Out(), nil
|
||||
}
|
||||
|
||||
func (b *Bus) Unsubscribe(address bus.Address, ch <-chan bus.Envelope) {
|
||||
func (b *Bus) Unsubscribe(ctx context.Context, ns bus.MessageNamespace, ch <-chan bus.Message) {
|
||||
logger.Debug(
|
||||
context.Background(), "unsubscribing",
|
||||
logger.F("address", address),
|
||||
ctx, "unsubscribing from messages",
|
||||
logger.F("messageNamespace", ns),
|
||||
)
|
||||
|
||||
dispatchers := b.getDispatchers(address)
|
||||
dispatchers := b.getDispatchers(ns)
|
||||
dispatchers.RemoveByOutChannel(ch)
|
||||
}
|
||||
|
||||
func (b *Bus) Publish(env bus.Envelope) error {
|
||||
dispatchers := b.getDispatchers(env.Address())
|
||||
func (b *Bus) Publish(ctx context.Context, msg bus.Message) error {
|
||||
dispatchers := b.getDispatchers(msg.MessageNamespace())
|
||||
dispatchersList := dispatchers.List()
|
||||
|
||||
logger.Debug(
|
||||
context.Background(), "publish",
|
||||
logger.F("address", env.Address()),
|
||||
ctx, "publishing message",
|
||||
logger.F("dispatchers", len(dispatchersList)),
|
||||
logger.F("messageNamespace", msg.MessageNamespace()),
|
||||
)
|
||||
|
||||
dispatchers.Range(func(d *eventDispatcher) {
|
||||
if err := d.In(env); err != nil {
|
||||
logger.Error(context.Background(), "could not publish message", logger.CapturedE(errors.WithStack(err)))
|
||||
for _, d := range dispatchersList {
|
||||
if d.Closed() {
|
||||
dispatchers.Remove(d)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := d.In(msg); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *Bus) getDispatchers(address bus.Address) *eventDispatcherSet {
|
||||
rawAddress := string(address)
|
||||
func (b *Bus) getDispatchers(namespace bus.MessageNamespace) *eventDispatcherSet {
|
||||
strNamespace := string(namespace)
|
||||
|
||||
rawDispatchers, exists := b.dispatchers.Get(rawAddress)
|
||||
rawDispatchers, exists := b.dispatchers.Get(strNamespace)
|
||||
dispatchers, ok := rawDispatchers.(*eventDispatcherSet)
|
||||
|
||||
if !exists || !ok {
|
||||
dispatchers = newEventDispatcherSet()
|
||||
b.dispatchers.Set(rawAddress, dispatchers)
|
||||
b.dispatchers.Set(strNamespace, dispatchers)
|
||||
}
|
||||
|
||||
return dispatchers
|
||||
|
||||
@ -4,23 +4,13 @@ import (
|
||||
"testing"
|
||||
|
||||
busTesting "forge.cadoles.com/arcad/edge/pkg/bus/testing"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
"go.uber.org/goleak"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
goleak.VerifyTestMain(m)
|
||||
}
|
||||
|
||||
func TestMemoryBus(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Test disabled when -short flag is set")
|
||||
}
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(logger.LevelDebug)
|
||||
}
|
||||
|
||||
t.Parallel()
|
||||
|
||||
t.Run("PublishSubscribe", func(t *testing.T) {
|
||||
@ -36,11 +26,4 @@ func TestMemoryBus(t *testing.T) {
|
||||
b := NewBus()
|
||||
busTesting.TestRequestReply(t, b)
|
||||
})
|
||||
|
||||
t.Run("CanceledRequestReply", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
b := NewBus()
|
||||
busTesting.TestCanceledRequest(t, b)
|
||||
})
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ package memory
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"github.com/pkg/errors"
|
||||
@ -29,7 +30,7 @@ func (s *eventDispatcherSet) Remove(d *eventDispatcher) {
|
||||
delete(s.items, d)
|
||||
}
|
||||
|
||||
func (s *eventDispatcherSet) RemoveByOutChannel(out <-chan bus.Envelope) {
|
||||
func (s *eventDispatcherSet) RemoveByOutChannel(out <-chan bus.Message) {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
@ -41,18 +42,17 @@ func (s *eventDispatcherSet) RemoveByOutChannel(out <-chan bus.Envelope) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *eventDispatcherSet) Range(fn func(d *eventDispatcher)) {
|
||||
func (s *eventDispatcherSet) List() []*eventDispatcher {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
dispatchers := make([]*eventDispatcher, 0, len(s.items))
|
||||
|
||||
for d := range s.items {
|
||||
if d.Closed() {
|
||||
s.Remove(d)
|
||||
continue
|
||||
dispatchers = append(dispatchers, d)
|
||||
}
|
||||
|
||||
fn(d)
|
||||
}
|
||||
return dispatchers
|
||||
}
|
||||
|
||||
func newEventDispatcherSet() *eventDispatcherSet {
|
||||
@ -62,8 +62,8 @@ func newEventDispatcherSet() *eventDispatcherSet {
|
||||
}
|
||||
|
||||
type eventDispatcher struct {
|
||||
in chan bus.Envelope
|
||||
out chan bus.Envelope
|
||||
in chan bus.Message
|
||||
out chan bus.Message
|
||||
mutex sync.RWMutex
|
||||
closed bool
|
||||
}
|
||||
@ -83,15 +83,11 @@ func (d *eventDispatcher) Close() {
|
||||
}
|
||||
|
||||
func (d *eventDispatcher) close() {
|
||||
if d.closed {
|
||||
return
|
||||
}
|
||||
|
||||
close(d.in)
|
||||
d.closed = true
|
||||
close(d.in)
|
||||
}
|
||||
|
||||
func (d *eventDispatcher) In(msg bus.Envelope) (err error) {
|
||||
func (d *eventDispatcher) In(msg bus.Message) (err error) {
|
||||
d.mutex.RLock()
|
||||
defer d.mutex.RUnlock()
|
||||
|
||||
@ -104,52 +100,67 @@ func (d *eventDispatcher) In(msg bus.Envelope) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *eventDispatcher) Out() <-chan bus.Envelope {
|
||||
func (d *eventDispatcher) Out() <-chan bus.Message {
|
||||
return d.out
|
||||
}
|
||||
|
||||
func (d *eventDispatcher) IsOut(out <-chan bus.Envelope) bool {
|
||||
func (d *eventDispatcher) IsOut(out <-chan bus.Message) bool {
|
||||
return d.out == out
|
||||
}
|
||||
|
||||
func (d *eventDispatcher) Run(ctx context.Context) {
|
||||
defer func() {
|
||||
for {
|
||||
logger.Debug(ctx, "closing dispatcher, flushing out incoming messages")
|
||||
|
||||
close(d.out)
|
||||
|
||||
for range d.in {
|
||||
// Flush all incoming messages
|
||||
for {
|
||||
_, ok := <-d.in
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
if err := ctx.Err(); !errors.Is(err, context.Canceled) {
|
||||
logger.Error(
|
||||
ctx,
|
||||
"message subscription context canceled",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
case msg, ok := <-d.in:
|
||||
msg, ok := <-d.in
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
d.out <- msg
|
||||
timeout := time.After(time.Second)
|
||||
|
||||
select {
|
||||
case d.out <- msg:
|
||||
case <-timeout:
|
||||
logger.Error(
|
||||
ctx,
|
||||
"out message channel timeout",
|
||||
logger.F("message", msg),
|
||||
)
|
||||
|
||||
return
|
||||
|
||||
case <-ctx.Done():
|
||||
logger.Error(
|
||||
ctx,
|
||||
"message subscription context canceled",
|
||||
logger.F("message", msg),
|
||||
logger.E(errors.WithStack(ctx.Err())),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newEventDispatcher(bufferSize int64) *eventDispatcher {
|
||||
return &eventDispatcher{
|
||||
in: make(chan bus.Envelope, bufferSize),
|
||||
out: make(chan bus.Envelope, bufferSize),
|
||||
in: make(chan bus.Message, bufferSize),
|
||||
out: make(chan bus.Message, bufferSize),
|
||||
closed: false,
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,78 +11,57 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AddressRequest bus.Address = "bus/memory/request"
|
||||
AddressReply bus.Address = "bus/memory/reply"
|
||||
MessageNamespaceRequest bus.MessageNamespace = "reqrep/request"
|
||||
MessageNamespaceReply bus.MessageNamespace = "reqrep/reply"
|
||||
)
|
||||
|
||||
type RequestEnvelope struct {
|
||||
requestID uint64
|
||||
wrapped bus.Envelope
|
||||
type RequestMessage struct {
|
||||
RequestID uint64
|
||||
|
||||
Message bus.Message
|
||||
|
||||
ns bus.MessageNamespace
|
||||
}
|
||||
|
||||
func (e *RequestEnvelope) Address() bus.Address {
|
||||
return getRequestAddress(e.wrapped.Address())
|
||||
func (m *RequestMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return m.ns
|
||||
}
|
||||
|
||||
func (e *RequestEnvelope) Message() any {
|
||||
return e.wrapped.Message()
|
||||
type ReplyMessage struct {
|
||||
RequestID uint64
|
||||
Message bus.Message
|
||||
Error error
|
||||
|
||||
ns bus.MessageNamespace
|
||||
}
|
||||
|
||||
func (e *RequestEnvelope) RequestID() uint64 {
|
||||
return e.requestID
|
||||
func (m *ReplyMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return m.ns
|
||||
}
|
||||
|
||||
func (e *RequestEnvelope) Unwrap() bus.Envelope {
|
||||
return e.wrapped
|
||||
}
|
||||
|
||||
type ReplyEnvelope struct {
|
||||
requestID uint64
|
||||
wrapped bus.Envelope
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *ReplyEnvelope) Address() bus.Address {
|
||||
return getReplyAddress(e.wrapped.Address(), e.requestID)
|
||||
}
|
||||
|
||||
func (e *ReplyEnvelope) Message() any {
|
||||
return e.wrapped.Message()
|
||||
}
|
||||
|
||||
func (e *ReplyEnvelope) Err() error {
|
||||
return e.err
|
||||
}
|
||||
|
||||
func (e *ReplyEnvelope) Unwrap() bus.Envelope {
|
||||
return e.wrapped
|
||||
}
|
||||
|
||||
func (b *Bus) Request(ctx context.Context, env bus.Envelope) (bus.Envelope, error) {
|
||||
func (b *Bus) Request(ctx context.Context, msg bus.Message) (bus.Message, error) {
|
||||
requestID := atomic.AddUint64(&b.nextRequestID, 1)
|
||||
|
||||
req := &RequestEnvelope{
|
||||
requestID: requestID,
|
||||
wrapped: env,
|
||||
req := &RequestMessage{
|
||||
RequestID: requestID,
|
||||
Message: msg,
|
||||
ns: msg.MessageNamespace(),
|
||||
}
|
||||
|
||||
replyAddress := getReplyAddress(env.Address(), requestID)
|
||||
replyNamespace := createReplyNamespace(requestID)
|
||||
|
||||
subCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
replies, err := b.Subscribe(subCtx, replyAddress)
|
||||
replies, err := b.Subscribe(ctx, replyNamespace)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
b.Unsubscribe(replyAddress, replies)
|
||||
b.Unsubscribe(ctx, replyNamespace, replies)
|
||||
}()
|
||||
|
||||
logger.Debug(ctx, "publishing request", logger.F("request", req))
|
||||
|
||||
if err := b.Publish(req); err != nil {
|
||||
if err := b.Publish(ctx, req); err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
@ -91,93 +70,82 @@ func (b *Bus) Request(ctx context.Context, env bus.Envelope) (bus.Envelope, erro
|
||||
case <-ctx.Done():
|
||||
return nil, errors.WithStack(ctx.Err())
|
||||
|
||||
case env, ok := <-replies:
|
||||
case msg, ok := <-replies:
|
||||
if !ok {
|
||||
return nil, errors.WithStack(bus.ErrNoResponse)
|
||||
}
|
||||
|
||||
reply, ok := env.(*ReplyEnvelope)
|
||||
reply, ok := msg.(*ReplyMessage)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(bus.ErrUnexpectedMessage)
|
||||
}
|
||||
|
||||
if err := reply.Err(); err != nil {
|
||||
if reply.Error != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return reply.Unwrap(), nil
|
||||
return reply.Message, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Bus) Reply(ctx context.Context, address bus.Address, handler bus.RequestHandler) chan error {
|
||||
requestAddress := getRequestAddress(address)
|
||||
type RequestHandler func(evt bus.Message) (bus.Message, error)
|
||||
|
||||
errs := make(chan error)
|
||||
|
||||
requests, err := b.Subscribe(ctx, requestAddress)
|
||||
func (b *Bus) Reply(ctx context.Context, msgNamespace bus.MessageNamespace, h bus.RequestHandler) error {
|
||||
requests, err := b.Subscribe(ctx, msgNamespace)
|
||||
if err != nil {
|
||||
go func() {
|
||||
errs <- errors.WithStack(err)
|
||||
close(errs)
|
||||
}()
|
||||
|
||||
return errs
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
defer func() {
|
||||
b.Unsubscribe(requestAddress, requests)
|
||||
close(errs)
|
||||
b.Unsubscribe(ctx, msgNamespace, requests)
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
errs <- errors.WithStack(ctx.Err())
|
||||
return
|
||||
return errors.WithStack(ctx.Err())
|
||||
|
||||
case env, ok := <-requests:
|
||||
case msg, ok := <-requests:
|
||||
if !ok {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
request, ok := env.(*RequestEnvelope)
|
||||
request, ok := msg.(*RequestMessage)
|
||||
if !ok {
|
||||
errs <- errors.WithStack(bus.ErrUnexpectedMessage)
|
||||
continue
|
||||
return errors.WithStack(bus.ErrUnexpectedMessage)
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "handling request", logger.F("request", request))
|
||||
|
||||
msg, err := handler(request.Unwrap())
|
||||
msg, err := h(request.Message)
|
||||
|
||||
reply := &ReplyEnvelope{
|
||||
requestID: request.RequestID(),
|
||||
wrapped: bus.NewEnvelope(request.Unwrap().Address(), msg),
|
||||
reply := &ReplyMessage{
|
||||
RequestID: request.RequestID,
|
||||
Message: nil,
|
||||
Error: nil,
|
||||
|
||||
ns: createReplyNamespace(request.RequestID),
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
reply.err = errors.WithStack(err)
|
||||
reply.Error = errors.WithStack(err)
|
||||
} else {
|
||||
reply.Message = msg
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "publishing reply", logger.F("reply", reply))
|
||||
|
||||
if err := b.Publish(reply); err != nil {
|
||||
errs <- errors.WithStack(err)
|
||||
continue
|
||||
if err := b.Publish(ctx, reply); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return errs
|
||||
}
|
||||
|
||||
func getRequestAddress(addr bus.Address) bus.Address {
|
||||
return AddressRequest + "/" + addr
|
||||
}
|
||||
|
||||
func getReplyAddress(addr bus.Address, requestID uint64) bus.Address {
|
||||
return AddressReply + "/" + addr + "/" + bus.Address(strconv.FormatUint(requestID, 10))
|
||||
func createReplyNamespace(requestID uint64) bus.MessageNamespace {
|
||||
return bus.NewMessageNamespace(
|
||||
MessageNamespaceReply,
|
||||
bus.MessageNamespace(strconv.FormatUint(requestID, 10)),
|
||||
)
|
||||
}
|
||||
|
||||
33
pkg/bus/message.go
Normal file
33
pkg/bus/message.go
Normal file
@ -0,0 +1,33 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type (
|
||||
MessageNamespace string
|
||||
)
|
||||
|
||||
type Message interface {
|
||||
MessageNamespace() MessageNamespace
|
||||
}
|
||||
|
||||
func NewMessageNamespace(namespaces ...MessageNamespace) MessageNamespace {
|
||||
var sb strings.Builder
|
||||
|
||||
for i, ns := range namespaces {
|
||||
if i != 0 {
|
||||
if _, err := sb.WriteString(":"); err != nil {
|
||||
panic(errors.Wrap(err, "could not build new message namespace"))
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := sb.WriteString(string(ns)); err != nil {
|
||||
panic(errors.Wrap(err, "could not build new message namespace"))
|
||||
}
|
||||
}
|
||||
|
||||
return MessageNamespace(sb.String())
|
||||
}
|
||||
@ -2,7 +2,6 @@ package testing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@ -13,52 +12,74 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
testAddress bus.Address = "testAddress"
|
||||
testNamespace bus.MessageNamespace = "testNamespace"
|
||||
)
|
||||
|
||||
type testMessage struct{}
|
||||
|
||||
func (e *testMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return testNamespace
|
||||
}
|
||||
|
||||
func TestPublishSubscribe(t *testing.T, b bus.Bus) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
t.Log("subscribe")
|
||||
|
||||
envelopes, err := b.Subscribe(ctx, testAddress)
|
||||
messages, err := b.Subscribe(ctx, testNamespace)
|
||||
if err != nil {
|
||||
t.Fatal(errors.WithStack(err))
|
||||
}
|
||||
|
||||
expectedTotal := 5
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Add(expectedTotal)
|
||||
wg.Add(5)
|
||||
|
||||
go func() {
|
||||
// 5 events should be received
|
||||
t.Log("publish 0")
|
||||
|
||||
count := expectedTotal
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
env := bus.NewEnvelope(testAddress, fmt.Sprintf("message %d", i))
|
||||
|
||||
if err := b.Publish(env); err != nil {
|
||||
if err := b.Publish(ctx, &testMessage{}); err != nil {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Logf("published %d", i)
|
||||
t.Log("publish 1")
|
||||
|
||||
if err := b.Publish(ctx, &testMessage{}); err != nil {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Log("publish 2")
|
||||
|
||||
if err := b.Publish(ctx, &testMessage{}); err != nil {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Log("publish 3")
|
||||
|
||||
if err := b.Publish(ctx, &testMessage{}); err != nil {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
|
||||
t.Log("publish 4")
|
||||
|
||||
if err := b.Publish(ctx, &testMessage{}); err != nil {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
}()
|
||||
|
||||
var count int32 = 0
|
||||
|
||||
go func() {
|
||||
t.Log("range for received envelopes")
|
||||
t.Log("range for events")
|
||||
|
||||
for env := range envelopes {
|
||||
for msg := range messages {
|
||||
t.Logf("received msg %d", atomic.LoadInt32(&count))
|
||||
atomic.AddInt32(&count, 1)
|
||||
|
||||
if e, g := testAddress, env.Address(); e != g {
|
||||
t.Errorf("env.Address(): expected '%v', got '%v'", e, g)
|
||||
if e, g := testNamespace, msg.MessageNamespace(); e != g {
|
||||
t.Errorf("evt.MessageNamespace(): expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
wg.Done()
|
||||
@ -67,9 +88,9 @@ func TestPublishSubscribe(t *testing.T, b bus.Bus) {
|
||||
|
||||
wg.Wait()
|
||||
|
||||
b.Unsubscribe(testAddress, envelopes)
|
||||
b.Unsubscribe(ctx, testNamespace, messages)
|
||||
|
||||
if e, g := int32(expectedTotal), count; e != g {
|
||||
t.Errorf("envelopes received count: expected '%v', got '%v'", e, g)
|
||||
if e, g := int32(5), count; e != g {
|
||||
t.Errorf("message received count: expected '%v', got '%v'", e, g)
|
||||
}
|
||||
}
|
||||
|
||||
@ -11,42 +11,58 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
testTypeReqResAddress bus.Address = "testTypeReqResAddress"
|
||||
testTypeReqRes bus.MessageNamespace = "testNamspaceReqRes"
|
||||
)
|
||||
|
||||
type testReqResMessage struct {
|
||||
i int
|
||||
}
|
||||
|
||||
func (m *testReqResMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return testNamespace
|
||||
}
|
||||
|
||||
func TestRequestReply(t *testing.T, b bus.Bus) {
|
||||
expectedRoundTrips := 256
|
||||
timeout := time.Now().Add(time.Duration(expectedRoundTrips) * time.Second)
|
||||
|
||||
replyCtx, cancelReply := context.WithDeadline(context.Background(), timeout)
|
||||
defer cancelReply()
|
||||
var (
|
||||
initWaitGroup sync.WaitGroup
|
||||
resWaitGroup sync.WaitGroup
|
||||
)
|
||||
|
||||
var resWaitGroup sync.WaitGroup
|
||||
initWaitGroup.Add(1)
|
||||
|
||||
replyErrs := b.Reply(replyCtx, testTypeReqResAddress, func(env bus.Envelope) (any, error) {
|
||||
go func() {
|
||||
repondCtx, cancelRespond := context.WithDeadline(context.Background(), timeout)
|
||||
defer cancelRespond()
|
||||
|
||||
initWaitGroup.Done()
|
||||
|
||||
err := b.Reply(repondCtx, testNamespace, func(msg bus.Message) (bus.Message, error) {
|
||||
defer resWaitGroup.Done()
|
||||
|
||||
req, ok := env.Message().(int)
|
||||
req, ok := msg.(*testReqResMessage)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(bus.ErrUnexpectedMessage)
|
||||
}
|
||||
|
||||
result := &testReqResMessage{req.i}
|
||||
|
||||
// Simulate random work
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
|
||||
t.Logf("[RES] sending res #%d", req)
|
||||
t.Logf("[RES] sending res #%d", req.i)
|
||||
|
||||
return req, nil
|
||||
return result, nil
|
||||
})
|
||||
|
||||
go func() {
|
||||
for err := range replyErrs {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
|
||||
initWaitGroup.Wait()
|
||||
|
||||
var reqWaitGroup sync.WaitGroup
|
||||
|
||||
for i := 0; i < expectedRoundTrips; i++ {
|
||||
@ -59,30 +75,32 @@ func TestRequestReply(t *testing.T, b bus.Bus) {
|
||||
requestCtx, cancelRequest := context.WithDeadline(context.Background(), timeout)
|
||||
defer cancelRequest()
|
||||
|
||||
req := &testReqResMessage{i}
|
||||
|
||||
t.Logf("[REQ] sending req #%d", i)
|
||||
|
||||
response, err := b.Request(requestCtx, bus.NewEnvelope(testTypeReqResAddress, i))
|
||||
result, err := b.Request(requestCtx, req)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
t.Logf("[REQ] received req #%d reply", i)
|
||||
|
||||
if response == nil {
|
||||
t.Error("response should not be nil")
|
||||
if result == nil {
|
||||
t.Error("result should not be nil")
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
result, ok := response.Message().(int)
|
||||
res, ok := result.(*testReqResMessage)
|
||||
if !ok {
|
||||
t.Error(errors.WithStack(bus.ErrUnexpectedMessage))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if e, g := i, result; e != g {
|
||||
t.Errorf("response.Message(): expected '%v', got '%v'", e, g)
|
||||
if e, g := req.i, res.i; e != g {
|
||||
t.Errorf("res.i: expected '%v', got '%v'", e, g)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
@ -90,77 +108,3 @@ func TestRequestReply(t *testing.T, b bus.Bus) {
|
||||
reqWaitGroup.Wait()
|
||||
resWaitGroup.Wait()
|
||||
}
|
||||
|
||||
func TestCanceledRequest(t *testing.T, b bus.Bus) {
|
||||
replyCtx, cancelReply := context.WithCancel(context.Background())
|
||||
defer cancelReply()
|
||||
|
||||
errs := b.Reply(replyCtx, testTypeReqResAddress, func(env bus.Envelope) (any, error) {
|
||||
return env.Message(), nil
|
||||
})
|
||||
|
||||
go func() {
|
||||
for err := range errs {
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
count := 100
|
||||
|
||||
wg.Add(count)
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
|
||||
t.Logf("calling %d", i)
|
||||
|
||||
isCanceled := i%2 == 0
|
||||
|
||||
var ctx context.Context
|
||||
if isCanceled {
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
ctx = canceledCtx
|
||||
} else {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
t.Logf("publishing envelope #%d", i)
|
||||
|
||||
reply, err := b.Request(ctx, bus.NewEnvelope(testTypeReqResAddress, int64(i)))
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) && isCanceled {
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, bus.ErrNoResponse) && isCanceled {
|
||||
return
|
||||
}
|
||||
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
result, ok := reply.Message().(int64)
|
||||
if !ok {
|
||||
t.Errorf("response.Result: expected type '%T', got '%T'", int64(0), reply.Message())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if e, g := i, int(result); e != g {
|
||||
t.Errorf("response.Result: expected '%v', got '%v'", e, g)
|
||||
|
||||
return
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
282
pkg/http/blob.go
Normal file
282
pkg/http/blob.go
Normal file
@ -0,0 +1,282 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/blob"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
errorCodeForbidden = "forbidden"
|
||||
errorCodeInternalError = "internal-error"
|
||||
errorCodeBadRequest = "bad-request"
|
||||
errorCodeNotFound = "not-found"
|
||||
)
|
||||
|
||||
type uploadResponse struct {
|
||||
Bucket string `json:"bucket"`
|
||||
BlobID storage.BlobID `json:"blobId"`
|
||||
}
|
||||
|
||||
func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
|
||||
h.mutex.RLock()
|
||||
defer h.mutex.RUnlock()
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, h.uploadMaxFileSize)
|
||||
|
||||
if err := r.ParseMultipartForm(h.uploadMaxFileSize); err != nil {
|
||||
logger.Error(ctx, "could not parse multipart form", logger.E(errors.WithStack(err)))
|
||||
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
_, fileHeader, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not read form file", logger.E(errors.WithStack(err)))
|
||||
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var metadata map[string]any
|
||||
|
||||
rawMetadata := r.Form.Get("metadata")
|
||||
if rawMetadata != "" {
|
||||
if err := json.Unmarshal([]byte(rawMetadata), &metadata); err != nil {
|
||||
logger.Error(ctx, "could not parse metadata", logger.E(errors.WithStack(err)))
|
||||
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx = module.WithContext(ctx, map[module.ContextKey]any{
|
||||
ContextKeyOriginRequest: r,
|
||||
})
|
||||
|
||||
requestMsg := blob.NewMessageUploadRequest(ctx, fileHeader, metadata)
|
||||
|
||||
reply, err := h.bus.Request(ctx, requestMsg)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve file", logger.E(errors.WithStack(err)))
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "upload reply", logger.F("reply", reply))
|
||||
|
||||
responseMsg, ok := reply.(*blob.MessageUploadResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected upload response message",
|
||||
logger.F("message", reply),
|
||||
)
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !responseMsg.Allow {
|
||||
jsonError(w, http.StatusForbidden, errorCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(w)
|
||||
res := &uploadResponse{
|
||||
Bucket: responseMsg.Bucket,
|
||||
BlobID: responseMsg.BlobID,
|
||||
}
|
||||
|
||||
if err := encoder.Encode(res); err != nil {
|
||||
panic(errors.Wrap(err, "could not encode upload response"))
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
|
||||
h.mutex.RLock()
|
||||
defer h.mutex.RUnlock()
|
||||
|
||||
bucket := chi.URLParam(r, "bucket")
|
||||
blobID := chi.URLParam(r, "blobID")
|
||||
|
||||
ctx := logger.With(r.Context(), logger.F("blobID", blobID), logger.F("bucket", bucket))
|
||||
ctx = module.WithContext(ctx, map[module.ContextKey]any{
|
||||
ContextKeyOriginRequest: r,
|
||||
})
|
||||
|
||||
requestMsg := blob.NewMessageDownloadRequest(ctx, bucket, storage.BlobID(blobID))
|
||||
|
||||
reply, err := h.bus.Request(ctx, requestMsg)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve file", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
replyMsg, ok := reply.(*blob.MessageDownloadResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected download response message",
|
||||
logger.E(errors.WithStack(bus.ErrUnexpectedMessage)),
|
||||
logger.F("message", reply),
|
||||
)
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !replyMsg.Allow {
|
||||
jsonError(w, http.StatusForbidden, errorCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if replyMsg.Blob == nil {
|
||||
jsonError(w, http.StatusNotFound, errorCodeNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := replyMsg.Blob.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close blob", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
http.ServeContent(w, r, string(replyMsg.BlobInfo.ID()), replyMsg.BlobInfo.ModTime(), replyMsg.Blob)
|
||||
}
|
||||
|
||||
func serveFile(w http.ResponseWriter, r *http.Request, fs fs.FS, path string) {
|
||||
ctx := logger.With(r.Context(), logger.F("path", path))
|
||||
|
||||
file, err := fs.Open(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(ctx, "error while opening fs file", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
logger.Error(ctx, "error while closing fs file", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
logger.Error(ctx, "error while retrieving fs file stat", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := file.(io.ReadSeeker)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeContent(w, r, path, info.ModTime(), reader)
|
||||
}
|
||||
|
||||
type jsonErrorResponse struct {
|
||||
Error jsonErr `json:"error"`
|
||||
}
|
||||
|
||||
type jsonErr struct {
|
||||
Code string `json:"code"`
|
||||
}
|
||||
|
||||
func jsonError(w http.ResponseWriter, status int, code string) {
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
|
||||
encoder := json.NewEncoder(w)
|
||||
response := jsonErrorResponse{
|
||||
Error: jsonErr{
|
||||
Code: code,
|
||||
},
|
||||
}
|
||||
|
||||
if err := encoder.Encode(response); err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
type uploadedFile struct {
|
||||
multipart.File
|
||||
header *multipart.FileHeader
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// Stat implements fs.File
|
||||
func (f *uploadedFile) Stat() (fs.FileInfo, error) {
|
||||
return &uploadedFileInfo{
|
||||
header: f.header,
|
||||
modTime: f.modTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type uploadedFileInfo struct {
|
||||
header *multipart.FileHeader
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// IsDir implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) IsDir() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ModTime implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) ModTime() time.Time {
|
||||
return i.modTime
|
||||
}
|
||||
|
||||
// Mode implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Mode() fs.FileMode {
|
||||
return os.ModePerm
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Name() string {
|
||||
return i.header.Filename
|
||||
}
|
||||
|
||||
// Size implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Size() int64 {
|
||||
return i.header.Size
|
||||
}
|
||||
|
||||
// Sys implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fs.File = &uploadedFile{}
|
||||
_ fs.FileInfo = &uploadedFileInfo{}
|
||||
)
|
||||
@ -7,11 +7,11 @@ import (
|
||||
)
|
||||
|
||||
func (h *Handler) handleSDKClient(w http.ResponseWriter, r *http.Request) {
|
||||
ServeFile(w, r, &sdk.FS, "client/dist/client.js")
|
||||
serveFile(w, r, &sdk.FS, "client/dist/client.js")
|
||||
}
|
||||
|
||||
func (h *Handler) handleSDKClientMap(w http.ResponseWriter, r *http.Request) {
|
||||
ServeFile(w, r, &sdk.FS, "client/dist/client.js.map")
|
||||
serveFile(w, r, &sdk.FS, "client/dist/client.js.map")
|
||||
}
|
||||
|
||||
func (h *Handler) handleAppFiles(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@ -1,75 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"net/http"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
)
|
||||
|
||||
type contextKey string
|
||||
|
||||
var (
|
||||
contextKeyBus contextKey = "bus"
|
||||
contextKeyHTTPRequest contextKey = "httpRequest"
|
||||
contextKeyHTTPClient contextKey = "httpClient"
|
||||
contextKeySessionID contextKey = "sessionId"
|
||||
)
|
||||
|
||||
func (h *Handler) contextMiddleware(next http.Handler) http.Handler {
|
||||
fn := func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
ctx = WithContextBus(ctx, h.bus)
|
||||
ctx = WithContextHTTPRequest(ctx, r)
|
||||
ctx = WithContextHTTPClient(ctx, h.httpClient)
|
||||
|
||||
r = r.WithContext(ctx)
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
return http.HandlerFunc(fn)
|
||||
}
|
||||
|
||||
func ContextBus(ctx context.Context) (bus.Bus, bool) {
|
||||
return contextValue[bus.Bus](ctx, contextKeyBus)
|
||||
}
|
||||
|
||||
func WithContextBus(parent context.Context, bus bus.Bus) context.Context {
|
||||
return context.WithValue(parent, contextKeyBus, bus)
|
||||
}
|
||||
|
||||
func ContextHTTPRequest(ctx context.Context) (*http.Request, bool) {
|
||||
return contextValue[*http.Request](ctx, contextKeyHTTPRequest)
|
||||
}
|
||||
|
||||
func WithContextHTTPRequest(parent context.Context, request *http.Request) context.Context {
|
||||
return context.WithValue(parent, contextKeyHTTPRequest, request)
|
||||
}
|
||||
|
||||
func ContextHTTPClient(ctx context.Context) (*http.Client, bool) {
|
||||
return contextValue[*http.Client](ctx, contextKeyHTTPClient)
|
||||
}
|
||||
|
||||
func WithContextHTTPClient(parent context.Context, client *http.Client) context.Context {
|
||||
return context.WithValue(parent, contextKeyHTTPClient, client)
|
||||
}
|
||||
|
||||
func ContextSessionID(ctx context.Context) (string, bool) {
|
||||
return contextValue[string](ctx, contextKeySessionID)
|
||||
}
|
||||
|
||||
func WithContextSessionID(parent context.Context, sessionID string) context.Context {
|
||||
return context.WithValue(parent, contextKeySessionID, sessionID)
|
||||
}
|
||||
|
||||
func contextValue[T any](ctx context.Context, key any) (T, bool) {
|
||||
value, ok := ctx.Value(key).(T)
|
||||
if !ok {
|
||||
return *new(T), false
|
||||
}
|
||||
|
||||
return value, true
|
||||
}
|
||||
@ -1,30 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
)
|
||||
|
||||
var (
|
||||
AddressIncomingMessage bus.Address = "http/incoming-message"
|
||||
AddressOutgoingMessage bus.Address = "http/outgoing-message"
|
||||
)
|
||||
|
||||
type IncomingMessage struct {
|
||||
Context context.Context
|
||||
Payload map[string]any
|
||||
}
|
||||
|
||||
func NewIncomingMessageEnvelope(ctx context.Context, payload map[string]any) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressIncomingMessage, &IncomingMessage{ctx, payload})
|
||||
}
|
||||
|
||||
type OutgoingMessage struct {
|
||||
SessionID string
|
||||
Data any
|
||||
}
|
||||
|
||||
func NewOutgoingMessageEnvelope(sessionID string, data any) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressOutgoingMessage, &OutgoingMessage{sessionID, data})
|
||||
}
|
||||
112
pkg/http/fetch.go
Normal file
112
pkg/http/fetch.go
Normal file
@ -0,0 +1,112 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
|
||||
h.mutex.RLock()
|
||||
defer h.mutex.RUnlock()
|
||||
|
||||
ctx := r.Context()
|
||||
|
||||
ctx = module.WithContext(ctx, map[module.ContextKey]any{
|
||||
ContextKeyOriginRequest: r,
|
||||
})
|
||||
|
||||
rawURL := r.URL.Query().Get("url")
|
||||
|
||||
url, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
requestMsg := fetch.NewMessageFetchRequest(ctx, r.RemoteAddr, url)
|
||||
|
||||
reply, err := h.bus.Request(ctx, requestMsg)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve fetch request reply", logger.E(errors.WithStack(err)))
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "fetch reply", logger.F("reply", reply))
|
||||
|
||||
responseMsg, ok := reply.(*fetch.MessageFetchResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected fetch response message",
|
||||
logger.F("message", reply),
|
||||
)
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !responseMsg.Allow {
|
||||
jsonError(w, http.StatusForbidden, errorCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
proxyReq, err := http.NewRequest(http.MethodGet, url.String(), nil)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not create proxy request",
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for header, values := range r.Header {
|
||||
for _, value := range values {
|
||||
proxyReq.Header.Add(header, value)
|
||||
}
|
||||
}
|
||||
|
||||
proxyReq.Header.Add("X-Forwarded-From", r.RemoteAddr)
|
||||
|
||||
res, err := h.httpClient.Do(proxyReq)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not execute proxy request",
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := res.Body.Close(); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not close response body",
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
}
|
||||
}()
|
||||
|
||||
for header, values := range res.Header {
|
||||
for _, value := range values {
|
||||
w.Header().Add(header, value)
|
||||
}
|
||||
}
|
||||
|
||||
w.WriteHeader(res.StatusCode)
|
||||
|
||||
if _, err := io.Copy(w, res.Body); err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,7 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"sync"
|
||||
|
||||
@ -27,6 +26,7 @@ type Handler struct {
|
||||
sockjs http.Handler
|
||||
bus bus.Bus
|
||||
sockjsOpts sockjs.Options
|
||||
uploadMaxFileSize int64
|
||||
|
||||
server *app.Server
|
||||
serverModuleFactories []app.ServerModuleFactory
|
||||
@ -40,7 +40,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
h.router.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) Load(ctx context.Context, bdle bundle.Bundle) error {
|
||||
func (h *Handler) Load(bdle bundle.Bundle) error {
|
||||
h.mutex.Lock()
|
||||
defer h.mutex.Unlock()
|
||||
|
||||
@ -49,13 +49,17 @@ func (h *Handler) Load(ctx context.Context, bdle bundle.Bundle) error {
|
||||
return errors.Wrap(err, "could not open server main script")
|
||||
}
|
||||
|
||||
mainScript, err := io.ReadAll(file)
|
||||
mainScript, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "could not read server main script")
|
||||
}
|
||||
|
||||
server := app.NewServer(h.serverModuleFactories...)
|
||||
|
||||
if err := server.Load(serverMainScript, string(mainScript)); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
fs := bundle.NewFileSystem("public", bdle)
|
||||
public := HTML5Fileserver(fs)
|
||||
sockjs := sockjs.NewHandler(sockJSPathPrefix, h.sockjsOpts, h.handleSockJSSession)
|
||||
@ -64,7 +68,7 @@ func (h *Handler) Load(ctx context.Context, bdle bundle.Bundle) error {
|
||||
h.server.Stop()
|
||||
}
|
||||
|
||||
if err := server.Start(ctx, serverMainScript, string(mainScript)); err != nil {
|
||||
if err := server.Start(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
@ -85,6 +89,7 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
|
||||
router := chi.NewRouter()
|
||||
|
||||
handler := &Handler{
|
||||
uploadMaxFileSize: opts.UploadMaxFileSize,
|
||||
sockjsOpts: opts.SockJS,
|
||||
router: router,
|
||||
serverModuleFactories: opts.ServerModuleFactories,
|
||||
@ -102,9 +107,15 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
|
||||
r.Get("/client.js.map", handler.handleSDKClientMap)
|
||||
})
|
||||
|
||||
r.Route("/api", func(r chi.Router) {
|
||||
r.Post("/v1/upload", handler.handleAppUpload)
|
||||
r.Get("/v1/download/{bucket}/{blobID}", handler.handleAppDownload)
|
||||
|
||||
r.Get("/v1/fetch", handler.handleAppFetch)
|
||||
})
|
||||
|
||||
for _, fn := range opts.HTTPMounts {
|
||||
r.Group(func(r chi.Router) {
|
||||
r.Use(handler.contextMiddleware)
|
||||
fn(r)
|
||||
})
|
||||
}
|
||||
|
||||
@ -27,10 +27,11 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler {
|
||||
r.URL.Path = "/"
|
||||
|
||||
handler.ServeHTTP(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(r.Context(), "could not open bundle file", logger.CapturedE(err))
|
||||
logger.Error(r.Context(), "could not open bundle file", logger.E(err))
|
||||
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
return
|
||||
@ -38,7 +39,7 @@ func HTML5Fileserver(fs http.FileSystem) http.Handler {
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
logger.Error(r.Context(), "could not close file", logger.CapturedE(err))
|
||||
logger.Error(r.Context(), "could not close file", logger.E(err))
|
||||
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
|
||||
@ -15,6 +15,7 @@ type HandlerOptions struct {
|
||||
Bus bus.Bus
|
||||
SockJS sockjs.Options
|
||||
ServerModuleFactories []app.ServerModuleFactory
|
||||
UploadMaxFileSize int64
|
||||
HTTPClient *http.Client
|
||||
HTTPMounts []func(r chi.Router)
|
||||
HTTPMiddlewares []func(next http.Handler) http.Handler
|
||||
@ -30,6 +31,7 @@ func defaultHandlerOptions() *HandlerOptions {
|
||||
Bus: memory.NewBus(),
|
||||
SockJS: sockjsOptions,
|
||||
ServerModuleFactories: make([]app.ServerModuleFactory, 0),
|
||||
UploadMaxFileSize: 10 << (10 * 2), // 10Mb
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: time.Second * 30,
|
||||
},
|
||||
@ -58,6 +60,12 @@ func WithBus(bus bus.Bus) HandlerOptionFunc {
|
||||
}
|
||||
}
|
||||
|
||||
func WithUploadMaxFileSize(size int64) HandlerOptionFunc {
|
||||
return func(opts *HandlerOptions) {
|
||||
opts.UploadMaxFileSize = size
|
||||
}
|
||||
}
|
||||
|
||||
func WithHTTPClient(client *http.Client) HandlerOptionFunc {
|
||||
return func(opts *HandlerOptions) {
|
||||
opts.HTTPClient = client
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"github.com/igm/sockjs-go/v3/sockjs"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
@ -14,6 +15,11 @@ const (
|
||||
statusChannelClosed = iota
|
||||
)
|
||||
|
||||
const (
|
||||
ContextKeySessionID module.ContextKey = "sessionId"
|
||||
ContextKeyOriginRequest module.ContextKey = "originRequest"
|
||||
)
|
||||
|
||||
func (h *Handler) handleSockJS(w http.ResponseWriter, r *http.Request) {
|
||||
h.mutex.RLock()
|
||||
defer h.mutex.RUnlock()
|
||||
@ -31,23 +37,24 @@ func (h *Handler) handleSockJSSession(sess sockjs.Session) {
|
||||
defer func() {
|
||||
if sess.GetSessionState() == sockjs.SessionActive {
|
||||
if err := sess.Close(statusChannelClosed, "channel closed"); err != nil {
|
||||
logger.Error(ctx, "could not close sockjs session", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close sockjs session", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go h.handleOutgoingMessages(ctx, sess)
|
||||
h.handleIncomingMessages(ctx, sess)
|
||||
go h.handleServerMessages(ctx, sess)
|
||||
h.handleClientMessages(ctx, sess)
|
||||
}
|
||||
|
||||
func (h *Handler) handleOutgoingMessages(ctx context.Context, sess sockjs.Session) {
|
||||
envelopes, err := h.bus.Subscribe(ctx, AddressOutgoingMessage)
|
||||
func (h *Handler) handleServerMessages(ctx context.Context, sess sockjs.Session) {
|
||||
messages, err := h.bus.Subscribe(ctx, module.MessageNamespaceServer)
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
h.bus.Unsubscribe(AddressOutgoingMessage, envelopes)
|
||||
// Close messages subscriber
|
||||
h.bus.Unsubscribe(ctx, module.MessageNamespaceServer, messages)
|
||||
|
||||
logger.Debug(ctx, "unsubscribed")
|
||||
|
||||
@ -56,7 +63,7 @@ func (h *Handler) handleOutgoingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
}
|
||||
|
||||
if err := sess.Close(statusChannelClosed, "channel closed"); err != nil {
|
||||
logger.Error(ctx, "could not close sockjs session", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close sockjs session", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -65,27 +72,31 @@ func (h *Handler) handleOutgoingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
case <-ctx.Done():
|
||||
return
|
||||
|
||||
case env := <-envelopes:
|
||||
outgoingMessage, ok := env.Message().(*OutgoingMessage)
|
||||
case msg := <-messages:
|
||||
serverMessage, ok := msg.(*module.ServerMessage)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx,
|
||||
"unexpected outgoing message",
|
||||
logger.F("message", env.Message()),
|
||||
"unexpected server message",
|
||||
logger.F("message", msg),
|
||||
)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
isDest := outgoingMessage.SessionID == "" || outgoingMessage.SessionID == sess.ID()
|
||||
sessionID := module.ContextValue[string](serverMessage.Context, ContextKeySessionID)
|
||||
|
||||
isDest := sessionID == "" || sessionID == sess.ID()
|
||||
if !isDest {
|
||||
continue
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(outgoingMessage.Data)
|
||||
payload, err := json.Marshal(serverMessage.Data)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not encode message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(err),
|
||||
)
|
||||
|
||||
continue
|
||||
@ -101,7 +112,7 @@ func (h *Handler) handleOutgoingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not encode message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(err),
|
||||
)
|
||||
|
||||
continue
|
||||
@ -114,14 +125,14 @@ func (h *Handler) handleOutgoingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not send message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) handleIncomingMessages(ctx context.Context, sess sockjs.Session) {
|
||||
func (h *Handler) handleClientMessages(ctx context.Context, sess sockjs.Session) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@ -134,14 +145,14 @@ func (h *Handler) handleIncomingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
|
||||
data, err := sess.RecvCtx(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, sockjs.ErrSessionNotOpen) || errors.Is(err, context.Canceled) {
|
||||
if errors.Is(err, sockjs.ErrSessionNotOpen) {
|
||||
break
|
||||
}
|
||||
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not read message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
|
||||
break
|
||||
@ -154,7 +165,7 @@ func (h *Handler) handleIncomingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not decode message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
|
||||
break
|
||||
@ -163,34 +174,38 @@ func (h *Handler) handleIncomingMessages(ctx context.Context, sess sockjs.Sessio
|
||||
switch {
|
||||
|
||||
case message.Type == WebsocketMessageTypeMessage:
|
||||
var payload map[string]any
|
||||
var payload map[string]interface{}
|
||||
if err := json.Unmarshal(message.Payload, &payload); err != nil {
|
||||
logger.Error(
|
||||
ctx,
|
||||
"could not decode payload",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(errors.WithStack(err)),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ctx := logger.With(ctx, logger.F("payload", payload))
|
||||
ctx = WithContextHTTPRequest(ctx, sess.Request())
|
||||
ctx = WithContextSessionID(ctx, sess.ID())
|
||||
ctx = module.WithContext(ctx, map[module.ContextKey]any{
|
||||
ContextKeySessionID: sess.ID(),
|
||||
ContextKeyOriginRequest: sess.Request(),
|
||||
})
|
||||
|
||||
incomingMessage := NewIncomingMessageEnvelope(ctx, payload)
|
||||
clientMessage := module.NewClientMessage(ctx, payload)
|
||||
|
||||
logger.Debug(ctx, "publishing new incoming message", logger.F("message", incomingMessage))
|
||||
logger.Debug(ctx, "publishing new client message", logger.F("message", clientMessage))
|
||||
|
||||
if err := h.bus.Publish(incomingMessage); err != nil {
|
||||
if err := h.bus.Publish(ctx, clientMessage); err != nil {
|
||||
logger.Error(ctx, "could not publish message",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.F("message", incomingMessage),
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("message", clientMessage),
|
||||
)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "new client message published", logger.F("message", clientMessage))
|
||||
|
||||
default:
|
||||
logger.Error(
|
||||
ctx,
|
||||
|
||||
@ -1,82 +0,0 @@
|
||||
package http
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
ErrCodeForbidden = "forbidden"
|
||||
ErrCodeInternalError = "internal-error"
|
||||
ErrCodeBadRequest = "bad-request"
|
||||
ErrCodeNotFound = "not-found"
|
||||
)
|
||||
|
||||
type jsonErrorResponse struct {
|
||||
Error jsonErr `json:"error"`
|
||||
}
|
||||
|
||||
type jsonErr struct {
|
||||
Code string `json:"code"`
|
||||
}
|
||||
|
||||
func JSONError(w http.ResponseWriter, status int, code string) {
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
|
||||
encoder := json.NewEncoder(w)
|
||||
response := jsonErrorResponse{
|
||||
Error: jsonErr{
|
||||
Code: code,
|
||||
},
|
||||
}
|
||||
|
||||
if err := encoder.Encode(response); err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
func ServeFile(w http.ResponseWriter, r *http.Request, fs fs.FS, path string) {
|
||||
ctx := logger.With(r.Context(), logger.F("path", path))
|
||||
|
||||
file, err := fs.Open(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(ctx, "error while opening fs file", logger.CapturedE(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
logger.Error(ctx, "error while closing fs file", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
info, err := file.Stat()
|
||||
if err != nil {
|
||||
logger.Error(ctx, "error while retrieving fs file stat", logger.CapturedE(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := file.(io.ReadSeeker)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
http.ServeContent(w, r, path, info.ModTime(), reader)
|
||||
}
|
||||
@ -1,36 +1,11 @@
|
||||
package jwtutil
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/lestrrat-go/jwx/v2/jwa"
|
||||
"github.com/lestrrat-go/jwx/v2/jwk"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func AddKeyWithSigningAlgo(keySet jwk.Set, key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm) error {
|
||||
addedKey := key
|
||||
|
||||
if !strings.HasPrefix(string(signingAlgorithm), "HS") {
|
||||
publicKey, err := key.PublicKey()
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
addedKey = publicKey
|
||||
}
|
||||
|
||||
if err := addedKey.Set(jwk.AlgorithmKey, signingAlgorithm); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
if err := keySet.AddKey(addedKey); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewKeySet(keys ...jwk.Key) (jwk.Set, error) {
|
||||
set := jwk.NewSet()
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/lestrrat-go/jwx/v2/jwk"
|
||||
"github.com/lestrrat-go/jwx/v2/jws"
|
||||
"github.com/lestrrat-go/jwx/v2/jwt"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
@ -110,7 +111,10 @@ func FindToken(r *http.Request, getKeySet GetKeySetFunc, funcs ...FindTokenOptio
|
||||
return nil, errors.WithStack(ErrNoKeySet)
|
||||
}
|
||||
|
||||
token, err := Parse([]byte(rawToken), keySet)
|
||||
token, err := jwt.Parse([]byte(rawToken),
|
||||
jwt.WithKeySet(keySet, jws.WithRequireKid(false)),
|
||||
jwt.WithValidate(true),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/lestrrat-go/jwx/v2/jwa"
|
||||
"github.com/lestrrat-go/jwx/v2/jwk"
|
||||
"github.com/lestrrat-go/jwx/v2/jws"
|
||||
"github.com/lestrrat-go/jwx/v2/jwt"
|
||||
"github.com/oklog/ulid/v2"
|
||||
"github.com/pkg/errors"
|
||||
@ -39,15 +38,3 @@ func SignedToken(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, claims ma
|
||||
|
||||
return rawToken, nil
|
||||
}
|
||||
|
||||
func Parse(rawToken []byte, keySet jwk.Set) (jwt.Token, error) {
|
||||
token, err := jwt.Parse(rawToken,
|
||||
jwt.WithKeySet(keySet, jws.WithRequireKid(false)),
|
||||
jwt.WithValidate(true),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ package memory
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"io/ioutil"
|
||||
"testing"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
@ -39,17 +39,20 @@ func TestAppModuleWithMemoryRepository(t *testing.T) {
|
||||
)),
|
||||
)
|
||||
|
||||
script := "testdata/app.js"
|
||||
file := "testdata/app.js"
|
||||
|
||||
data, err := os.ReadFile(script)
|
||||
data, err := ioutil.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
if err := server.Load(file, string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer server.Stop()
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,7 +20,7 @@ type Handler struct {
|
||||
func (h *Handler) serveApps(w http.ResponseWriter, r *http.Request) {
|
||||
manifests, err := h.repo.List(r.Context())
|
||||
if err != nil {
|
||||
logger.Error(r.Context(), "could not retrieve app manifest", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(r.Context(), "could not retrieve app manifest", logger.E(errors.WithStack(err)))
|
||||
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
|
||||
|
||||
return
|
||||
@ -44,7 +44,7 @@ func (h *Handler) serveApp(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(r.Context(), "could not retrieve app manifest", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(r.Context(), "could not retrieve app manifest", logger.E(errors.WithStack(err)))
|
||||
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
|
||||
|
||||
return
|
||||
@ -84,7 +84,7 @@ func (h *Handler) serveAppURL(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(r.Context(), "could not retrieve app url", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(r.Context(), "could not retrieve app url", logger.E(errors.WithStack(err)))
|
||||
api.ErrorResponse(w, http.StatusInternalServerError, api.ErrCodeUnknownError, nil)
|
||||
|
||||
return
|
||||
|
||||
@ -69,7 +69,7 @@ func (h *LocalHandler) serveForm(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
if err := loginTemplate.Execute(w, data); err != nil {
|
||||
logger.Error(ctx, "could not execute login page template", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not execute login page template", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}
|
||||
|
||||
@ -77,7 +77,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
if err := r.ParseForm(); err != nil {
|
||||
logger.Error(ctx, "could not parse form", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not parse form", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
|
||||
|
||||
return
|
||||
@ -99,13 +99,13 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
|
||||
data.Message = "Invalid username or password."
|
||||
|
||||
if err := loginTemplate.Execute(w, data); err != nil {
|
||||
logger.Error(ctx, "could not execute login page template", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not execute login page template", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(ctx, "could not authenticate account", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not authenticate account", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -115,7 +115,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
token, err := jwtutil.SignedToken(h.key, h.signingAlgorithm, account.Claims)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not generate signed token", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -123,7 +123,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
cookieDomain, err := h.getCookieDomain(r)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -146,7 +146,7 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *LocalHandler) handleLogout(w http.ResponseWriter, r *http.Request) {
|
||||
cookieDomain, err := h.getCookieDomain(r)
|
||||
if err != nil {
|
||||
logger.Error(r.Context(), "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(r.Context(), "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
|
||||
@ -42,7 +42,7 @@ func AnonymousUser(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs .
|
||||
|
||||
uuid, err := uuid.NewUUID()
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not generate uuid for anonymous user", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not generate uuid for anonymous user", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -51,7 +51,7 @@ func AnonymousUser(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs .
|
||||
subject := fmt.Sprintf("%s-%s", AnonIssuer, uuid.String())
|
||||
preferredUsername, err := generateRandomPreferredUsername(8)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not generate preferred username for anonymous user", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not generate preferred username for anonymous user", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -68,7 +68,7 @@ func AnonymousUser(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs .
|
||||
|
||||
token, err := jwtutil.SignedToken(key, signingAlgorithm, claims)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not generate signed token", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not generate signed token", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
@ -76,7 +76,7 @@ func AnonymousUser(key jwk.Key, signingAlgorithm jwa.SignatureAlgorithm, funcs .
|
||||
|
||||
cookieDomain, err := opts.GetCookieDomain(r)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve cookie domain", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/util"
|
||||
"github.com/dop251/goja"
|
||||
@ -66,7 +68,7 @@ func (m *Module) getClaim(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
ctx := util.AssertContext(call.Argument(0), rt)
|
||||
claimName := util.AssertString(call.Argument(1), rt)
|
||||
|
||||
req, ok := edgehttp.ContextHTTPRequest(ctx)
|
||||
req, ok := ctx.Value(edgeHTTP.ContextKeyOriginRequest).(*http.Request)
|
||||
if !ok {
|
||||
panic(rt.ToValue(errors.New("could not find http request in context")))
|
||||
}
|
||||
@ -77,7 +79,7 @@ func (m *Module) getClaim(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.Error(ctx, "could not retrieve claim", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not retrieve claim", logger.E(errors.WithStack(err)))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -2,14 +2,14 @@ package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"cdr.dev/slog"
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"forge.cadoles.com/arcad/edge/pkg/jwtutil"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"github.com/lestrrat-go/jwx/v2/jwa"
|
||||
@ -22,9 +22,7 @@ import (
|
||||
func TestAuthModule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
key := getDummyKey()
|
||||
|
||||
@ -35,15 +33,16 @@ func TestAuthModule(t *testing.T) {
|
||||
),
|
||||
)
|
||||
|
||||
script := "testdata/auth.js"
|
||||
|
||||
data, err := os.ReadFile(script)
|
||||
data, err := ioutil.ReadFile("testdata/auth.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
if err := server.Load("testdata/auth.js", string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
@ -71,7 +70,7 @@ func TestAuthModule(t *testing.T) {
|
||||
|
||||
req.Header.Add("Authorization", "Bearer "+string(rawToken))
|
||||
|
||||
ctx = edgehttp.WithContextHTTPRequest(context.Background(), req)
|
||||
ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
|
||||
|
||||
if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
@ -81,9 +80,7 @@ func TestAuthModule(t *testing.T) {
|
||||
func TestAuthAnonymousModule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
key := getDummyKey()
|
||||
|
||||
@ -92,15 +89,16 @@ func TestAuthAnonymousModule(t *testing.T) {
|
||||
ModuleFactory(WithJWT(getDummyKeySet(key))),
|
||||
)
|
||||
|
||||
script := "testdata/auth_anonymous.js"
|
||||
|
||||
data, err := os.ReadFile("testdata/auth_anonymous.js")
|
||||
data, err := ioutil.ReadFile("testdata/auth_anonymous.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
if err := server.Load("testdata/auth_anonymous.js", string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
@ -111,7 +109,7 @@ func TestAuthAnonymousModule(t *testing.T) {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
ctx = edgehttp.WithContextHTTPRequest(context.Background(), req)
|
||||
ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
|
||||
|
||||
if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
|
||||
@ -35,7 +35,7 @@ func (h *Handler) serveProfile(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Error(ctx, "could not retrieve claims", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not retrieve claims", logger.E(errors.WithStack(err)))
|
||||
api.ErrorResponse(
|
||||
w, http.StatusInternalServerError,
|
||||
api.ErrCodeUnknownError,
|
||||
|
||||
92
pkg/module/blob/blob_message.go
Normal file
92
pkg/module/blob/blob_message.go
Normal file
@ -0,0 +1,92 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"github.com/oklog/ulid/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
MessageNamespaceUploadRequest bus.MessageNamespace = "uploadRequest"
|
||||
MessageNamespaceUploadResponse bus.MessageNamespace = "uploadResponse"
|
||||
MessageNamespaceDownloadRequest bus.MessageNamespace = "downloadRequest"
|
||||
MessageNamespaceDownloadResponse bus.MessageNamespace = "downloadResponse"
|
||||
)
|
||||
|
||||
type MessageUploadRequest struct {
|
||||
Context context.Context
|
||||
RequestID string
|
||||
FileHeader *multipart.FileHeader
|
||||
Metadata map[string]interface{}
|
||||
}
|
||||
|
||||
func (m *MessageUploadRequest) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceUploadRequest
|
||||
}
|
||||
|
||||
func NewMessageUploadRequest(ctx context.Context, fileHeader *multipart.FileHeader, metadata map[string]interface{}) *MessageUploadRequest {
|
||||
return &MessageUploadRequest{
|
||||
Context: ctx,
|
||||
RequestID: ulid.Make().String(),
|
||||
FileHeader: fileHeader,
|
||||
Metadata: metadata,
|
||||
}
|
||||
}
|
||||
|
||||
type MessageUploadResponse struct {
|
||||
RequestID string
|
||||
BlobID storage.BlobID
|
||||
Bucket string
|
||||
Allow bool
|
||||
}
|
||||
|
||||
func (m *MessageUploadResponse) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceDownloadResponse
|
||||
}
|
||||
|
||||
func NewMessageUploadResponse(requestID string) *MessageUploadResponse {
|
||||
return &MessageUploadResponse{
|
||||
RequestID: requestID,
|
||||
}
|
||||
}
|
||||
|
||||
type MessageDownloadRequest struct {
|
||||
Context context.Context
|
||||
RequestID string
|
||||
Bucket string
|
||||
BlobID storage.BlobID
|
||||
}
|
||||
|
||||
func (m *MessageDownloadRequest) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceDownloadRequest
|
||||
}
|
||||
|
||||
func NewMessageDownloadRequest(ctx context.Context, bucket string, blobID storage.BlobID) *MessageDownloadRequest {
|
||||
return &MessageDownloadRequest{
|
||||
Context: ctx,
|
||||
RequestID: ulid.Make().String(),
|
||||
Bucket: bucket,
|
||||
BlobID: blobID,
|
||||
}
|
||||
}
|
||||
|
||||
type MessageDownloadResponse struct {
|
||||
RequestID string
|
||||
Allow bool
|
||||
BlobInfo storage.BlobInfo
|
||||
Blob io.ReadSeekCloser
|
||||
}
|
||||
|
||||
func (m *MessageDownloadResponse) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceDownloadResponse
|
||||
}
|
||||
|
||||
func NewMessageDownloadResponse(requestID string) *MessageDownloadResponse {
|
||||
return &MessageDownloadResponse{
|
||||
RequestID: requestID,
|
||||
}
|
||||
}
|
||||
@ -1,55 +0,0 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
)
|
||||
|
||||
const (
|
||||
AddressUpload bus.Address = "module/blob/upload"
|
||||
AddressDownload bus.Address = "module/blob/download"
|
||||
)
|
||||
|
||||
type UploadRequest struct {
|
||||
Context context.Context
|
||||
FileHeader *multipart.FileHeader
|
||||
Metadata map[string]interface{}
|
||||
}
|
||||
|
||||
func NewUploadRequestEnvelope(ctx context.Context, fileHeader *multipart.FileHeader, metadata map[string]interface{}) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressUpload, &UploadRequest{
|
||||
Context: ctx,
|
||||
FileHeader: fileHeader,
|
||||
Metadata: metadata,
|
||||
})
|
||||
}
|
||||
|
||||
type UploadResponse struct {
|
||||
Allow bool
|
||||
Bucket string
|
||||
BlobID storage.BlobID
|
||||
}
|
||||
|
||||
type DownloadRequest struct {
|
||||
Context context.Context
|
||||
Bucket string
|
||||
BlobID storage.BlobID
|
||||
}
|
||||
|
||||
func NewDownloadRequestEnvelope(ctx context.Context, bucket string, blobID storage.BlobID) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressDownload, &DownloadRequest{
|
||||
Context: ctx,
|
||||
Bucket: bucket,
|
||||
BlobID: blobID,
|
||||
})
|
||||
}
|
||||
|
||||
type DownloadResponse struct {
|
||||
Allow bool
|
||||
Blob io.ReadSeekCloser
|
||||
BlobInfo storage.BlobInfo
|
||||
}
|
||||
@ -1,230 +0,0 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/fs"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"forge.cadoles.com/arcad/edge/pkg/storage"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type uploadResponse struct {
|
||||
Bucket string `json:"bucket"`
|
||||
BlobID storage.BlobID `json:"blobId"`
|
||||
}
|
||||
|
||||
func Mount(uploadMaxFileSize int64) func(r chi.Router) {
|
||||
return func(r chi.Router) {
|
||||
r.Post("/api/v1/upload", getAppUploadHandler(uploadMaxFileSize))
|
||||
r.Get("/api/v1/download/{bucket}/{blobID}", handleAppDownload)
|
||||
}
|
||||
}
|
||||
|
||||
func getAppUploadHandler(uploadMaxFileSize int64) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
r.Body = http.MaxBytesReader(w, r.Body, uploadMaxFileSize)
|
||||
|
||||
if err := r.ParseMultipartForm(uploadMaxFileSize); err != nil {
|
||||
logger.Error(ctx, "could not parse multipart form", logger.CapturedE(errors.WithStack(err)))
|
||||
edgehttp.JSONError(w, http.StatusBadRequest, edgehttp.ErrCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
_, fileHeader, err := r.FormFile("file")
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not read form file", logger.CapturedE(errors.WithStack(err)))
|
||||
edgehttp.JSONError(w, http.StatusBadRequest, edgehttp.ErrCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
var metadata map[string]any
|
||||
|
||||
rawMetadata := r.Form.Get("metadata")
|
||||
if rawMetadata != "" {
|
||||
if err := json.Unmarshal([]byte(rawMetadata), &metadata); err != nil {
|
||||
logger.Error(ctx, "could not parse metadata", logger.CapturedE(errors.WithStack(err)))
|
||||
edgehttp.JSONError(w, http.StatusBadRequest, edgehttp.ErrCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
requestEnv := NewUploadRequestEnvelope(ctx, fileHeader, metadata)
|
||||
|
||||
bus, ok := edgehttp.ContextBus(ctx)
|
||||
if !ok {
|
||||
logger.Error(ctx, "could find bus on context")
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
reply, err := bus.Request(ctx, requestEnv)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve file", logger.CapturedE(errors.WithStack(err)))
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "upload reply", logger.F("reply", reply))
|
||||
|
||||
replyMessage, ok := reply.Message().(*UploadResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected upload response message",
|
||||
logger.F("message", reply.Message()),
|
||||
)
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !replyMessage.Allow {
|
||||
edgehttp.JSONError(w, http.StatusForbidden, edgehttp.ErrCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(w)
|
||||
res := &uploadResponse{
|
||||
Bucket: replyMessage.Bucket,
|
||||
BlobID: replyMessage.BlobID,
|
||||
}
|
||||
|
||||
if err := encoder.Encode(res); err != nil {
|
||||
panic(errors.Wrap(err, "could not encode upload response"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleAppDownload(w http.ResponseWriter, r *http.Request) {
|
||||
bucket := chi.URLParam(r, "bucket")
|
||||
blobID := chi.URLParam(r, "blobID")
|
||||
|
||||
ctx := logger.With(r.Context(), logger.F("blobID", blobID), logger.F("bucket", bucket))
|
||||
|
||||
requestMsg := NewDownloadRequestEnvelope(ctx, bucket, storage.BlobID(blobID))
|
||||
|
||||
bs, ok := edgehttp.ContextBus(ctx)
|
||||
if !ok {
|
||||
logger.Error(ctx, "could find bus on context")
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
reply, err := bs.Request(ctx, requestMsg)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve file", logger.CapturedE(errors.WithStack(err)))
|
||||
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
replyMessage, ok := reply.Message().(*DownloadResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected download response message",
|
||||
logger.CapturedE(errors.WithStack(bus.ErrUnexpectedMessage)),
|
||||
logger.F("message", reply),
|
||||
)
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !replyMessage.Allow {
|
||||
edgehttp.JSONError(w, http.StatusForbidden, edgehttp.ErrCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if replyMessage.Blob == nil {
|
||||
edgehttp.JSONError(w, http.StatusNotFound, edgehttp.ErrCodeNotFound)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := replyMessage.Blob.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close blob", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
// TODO Fix usage of ServeContent
|
||||
// http.ServeContent(w, r, string(replyMessage.BlobInfo.ID()), replyMessage.BlobInfo.ModTime(), replyMessage.Blob)
|
||||
|
||||
w.Header().Add("Content-Type", replyMessage.BlobInfo.ContentType())
|
||||
|
||||
if _, err := io.Copy(w, replyMessage.Blob); err != nil {
|
||||
logger.Error(ctx, "could not write blob", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}
|
||||
|
||||
type uploadedFile struct {
|
||||
multipart.File
|
||||
header *multipart.FileHeader
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// Stat implements fs.File
|
||||
func (f *uploadedFile) Stat() (fs.FileInfo, error) {
|
||||
return &uploadedFileInfo{
|
||||
header: f.header,
|
||||
modTime: f.modTime,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type uploadedFileInfo struct {
|
||||
header *multipart.FileHeader
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// IsDir implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) IsDir() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ModTime implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) ModTime() time.Time {
|
||||
return i.modTime
|
||||
}
|
||||
|
||||
// Mode implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Mode() fs.FileMode {
|
||||
return os.ModePerm
|
||||
}
|
||||
|
||||
// Name implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Name() string {
|
||||
return i.header.Filename
|
||||
}
|
||||
|
||||
// Size implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Size() int64 {
|
||||
return i.header.Size
|
||||
}
|
||||
|
||||
// Sys implements fs.FileInfo
|
||||
func (i *uploadedFileInfo) Sys() any {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ fs.File = &uploadedFile{}
|
||||
_ fs.FileInfo = &uploadedFileInfo{}
|
||||
)
|
||||
@ -95,7 +95,7 @@ func (m *Module) writeBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value
|
||||
|
||||
defer func() {
|
||||
if err := bucket.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -106,7 +106,7 @@ func (m *Module) writeBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value
|
||||
|
||||
defer func() {
|
||||
if err := writer.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||
logger.Error(ctx, "could not close blob writer", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close blob writer", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -129,7 +129,7 @@ func (m *Module) getBlobInfo(call goja.FunctionCall, rt *goja.Runtime) goja.Valu
|
||||
|
||||
defer func() {
|
||||
if err := bucket.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -153,7 +153,7 @@ func (m *Module) readBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
|
||||
defer func() {
|
||||
if err := reader.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||
logger.Error(ctx, "could not close blob reader", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close blob reader", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -236,15 +236,16 @@ func (m *Module) getBucketSize(call goja.FunctionCall, rt *goja.Runtime) goja.Va
|
||||
func (m *Module) handleMessages() {
|
||||
ctx := context.Background()
|
||||
|
||||
uploadRequestErrs := m.bus.Reply(ctx, AddressUpload, func(env bus.Envelope) (any, error) {
|
||||
uploadRequest, ok := env.Message().(*UploadRequest)
|
||||
go func() {
|
||||
err := m.bus.Reply(ctx, MessageNamespaceUploadRequest, func(msg bus.Message) (bus.Message, error) {
|
||||
uploadRequest, ok := msg.(*MessageUploadRequest)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message upload request, got '%T'", env.Message())
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message upload request, got '%T'", msg)
|
||||
}
|
||||
|
||||
res, err := m.handleUploadRequest(uploadRequest)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not handle upload request", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not handle upload request", logger.E(errors.WithStack(err)))
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
@ -253,37 +254,34 @@ func (m *Module) handleMessages() {
|
||||
|
||||
return res, nil
|
||||
})
|
||||
|
||||
go func() {
|
||||
for err := range uploadRequestErrs {
|
||||
logger.Error(ctx, "error while replying to upload requests", logger.CapturedE(errors.WithStack(err)))
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}()
|
||||
|
||||
downloadRequestErrs := m.bus.Reply(ctx, AddressDownload, func(env bus.Envelope) (any, error) {
|
||||
downloadRequest, ok := env.Message().(*DownloadRequest)
|
||||
err := m.bus.Reply(ctx, MessageNamespaceDownloadRequest, func(msg bus.Message) (bus.Message, error) {
|
||||
downloadRequest, ok := msg.(*MessageDownloadRequest)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message download request, got '%T'", env.Message())
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message download request, got '%T'", msg)
|
||||
}
|
||||
|
||||
res, err := m.handleDownloadRequest(downloadRequest)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not handle download request", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not handle download request", logger.E(errors.WithStack(err)))
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return res, nil
|
||||
})
|
||||
|
||||
for err := range downloadRequestErrs {
|
||||
logger.Fatal(ctx, "error while replying to download requests", logger.CapturedE(errors.WithStack(err)))
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) handleUploadRequest(req *UploadRequest) (*UploadResponse, error) {
|
||||
func (m *Module) handleUploadRequest(req *MessageUploadRequest) (*MessageUploadResponse, error) {
|
||||
blobID := storage.NewBlobID()
|
||||
res := &UploadResponse{}
|
||||
res := NewMessageUploadResponse(req.RequestID)
|
||||
|
||||
ctx := logger.With(req.Context, logger.F("blobID", blobID))
|
||||
|
||||
@ -304,11 +302,11 @@ func (m *Module) handleUploadRequest(req *UploadRequest) (*UploadResponse, error
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
result, ok := rawResult.(map[string]interface{})
|
||||
result, ok := rawResult.Export().(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.Errorf(
|
||||
"unexpected onBlobUpload result: expected 'map[string]interface{}', got '%T'",
|
||||
rawResult,
|
||||
rawResult.Export(),
|
||||
)
|
||||
}
|
||||
|
||||
@ -356,7 +354,7 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close file", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -367,7 +365,7 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
|
||||
|
||||
defer func() {
|
||||
if err := bucket.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -378,13 +376,13 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
|
||||
|
||||
defer func() {
|
||||
if err := file.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close file", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
defer func() {
|
||||
if err := writer.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close writer", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not close writer", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
@ -395,8 +393,8 @@ func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) handleDownloadRequest(req *DownloadRequest) (*DownloadResponse, error) {
|
||||
res := &DownloadResponse{}
|
||||
func (m *Module) handleDownloadRequest(req *MessageDownloadRequest) (*MessageDownloadResponse, error) {
|
||||
res := NewMessageDownloadResponse(req.RequestID)
|
||||
|
||||
rawResult, err := m.server.ExecFuncByName(req.Context, "onBlobDownload", req.Context, req.Bucket, req.BlobID)
|
||||
if err != nil {
|
||||
@ -409,11 +407,11 @@ func (m *Module) handleDownloadRequest(req *DownloadRequest) (*DownloadResponse,
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
result, ok := rawResult.(map[string]interface{})
|
||||
result, ok := rawResult.Export().(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.Errorf(
|
||||
"unexpected onBlobDownload result: expected 'map[string]interface{}', got '%T'",
|
||||
rawResult,
|
||||
rawResult.Export(),
|
||||
)
|
||||
}
|
||||
|
||||
@ -455,7 +453,7 @@ func (m *Module) openBlob(ctx context.Context, bucketName string, blobID storage
|
||||
|
||||
defer func() {
|
||||
if err := bucket.Close(); err != nil {
|
||||
logger.Error(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)), logger.F("bucket", bucket))
|
||||
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)), logger.F("bucket", bucket))
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package blob
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@ -17,9 +16,7 @@ import (
|
||||
func TestBlobModule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
bus := memory.NewBus()
|
||||
store := sqlite.NewBlobStore(":memory:?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000")
|
||||
@ -30,17 +27,18 @@ func TestBlobModule(t *testing.T) {
|
||||
ModuleFactory(bus, store),
|
||||
)
|
||||
|
||||
script := "testdata/blob.js"
|
||||
|
||||
data, err := os.ReadFile(script)
|
||||
data, err := os.ReadFile("testdata/blob.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
if err := server.Load("testdata/blob.js", string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
defer server.Stop()
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
@ -148,7 +148,7 @@ func SearchDevices(ctx context.Context) (chan Device, error) {
|
||||
defer searchDevicesMutex.Unlock()
|
||||
|
||||
if err := service.Run(ctx, serviceDiscoveryPollingInterval); err != nil && !errors.Is(err, context.DeadlineExceeded) {
|
||||
logger.Error(ctx, "error while running cast service discovery", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "error while running cast service discovery", logger.E(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
|
||||
@ -21,9 +21,7 @@ func TestCastLoadURL(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@ -71,7 +71,7 @@ func (d *Service) Run(ctx context.Context, interval time.Duration) error {
|
||||
|
||||
logger.Error(
|
||||
ctx, "could not poll interface",
|
||||
logger.CapturedE(errors.WithStack(err)), logger.F("iface", iface.Name),
|
||||
logger.E(errors.WithStack(err)), logger.F("iface", iface.Name),
|
||||
)
|
||||
}
|
||||
}(pollCtx, iface)
|
||||
|
||||
@ -60,7 +60,7 @@ func (m *Module) refreshDevices(call goja.FunctionCall, rt *goja.Runtime) goja.V
|
||||
devices, err := ListDevices(ctx, true)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Error(ctx, "error refreshing casting devices list", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "error refreshing casting devices list", logger.E(errors.WithStack(err)))
|
||||
|
||||
promise.Reject(err)
|
||||
|
||||
@ -108,7 +108,7 @@ func (m *Module) loadUrl(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
err := LoadURL(ctx, deviceUUID, url)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Error(ctx, "error while casting url", logger.CapturedE(err))
|
||||
logger.Error(ctx, "error while casting url", logger.E(err))
|
||||
|
||||
promise.Reject(err)
|
||||
|
||||
@ -143,7 +143,7 @@ func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
err := StopCast(ctx, deviceUUID)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Error(ctx, "error while quitting casting device app", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "error while quitting casting device app", logger.E(errors.WithStack(err)))
|
||||
|
||||
promise.Reject(err)
|
||||
|
||||
@ -178,7 +178,7 @@ func (m *Module) getStatus(call goja.FunctionCall, rt *goja.Runtime) goja.Value
|
||||
status, err := getStatus(ctx, deviceUUID)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Error(ctx, "error while getting casting device status", logger.CapturedE(err))
|
||||
logger.Error(ctx, "error while getting casting device status", logger.E(err))
|
||||
|
||||
promise.Reject(err)
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@ package cast
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
@ -23,24 +24,23 @@ func TestCastModule(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
server := app.NewServer(
|
||||
module.ConsoleModuleFactory(),
|
||||
CastModuleFactory(),
|
||||
)
|
||||
|
||||
script := "testdata/cast.js"
|
||||
|
||||
data, err := os.ReadFile(script)
|
||||
data, err := ioutil.ReadFile("testdata/cast.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
if err := server.Load("testdata/cast.js", string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
@ -58,24 +58,23 @@ func TestCastModuleRefreshDevices(t *testing.T) {
|
||||
return
|
||||
}
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
server := app.NewServer(
|
||||
module.ConsoleModuleFactory(),
|
||||
CastModuleFactory(),
|
||||
)
|
||||
|
||||
script := "testdata/refresh_devices.js"
|
||||
|
||||
data, err := os.ReadFile(script)
|
||||
data, err := ioutil.ReadFile("testdata/refresh_devices.js")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, script, string(data)); err != nil {
|
||||
if err := server.Load("testdata/refresh_devices.js", string(data)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
@ -86,5 +85,12 @@ func TestCastModuleRefreshDevices(t *testing.T) {
|
||||
t.Error(errors.WithStack(err))
|
||||
}
|
||||
|
||||
spew.Dump(result)
|
||||
promise, ok := app.IsPromise(result)
|
||||
if !ok {
|
||||
t.Fatal("expected promise")
|
||||
}
|
||||
|
||||
value := server.WaitForPromise(promise)
|
||||
|
||||
spew.Dump(value.Export())
|
||||
}
|
||||
|
||||
@ -1,38 +0,0 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
AddressFetchRequest bus.Address = "module/fetch/request"
|
||||
AddressFetchResponse bus.Address = "module/fetch/response"
|
||||
)
|
||||
|
||||
type FetchRequest struct {
|
||||
Context context.Context
|
||||
RequestID string
|
||||
URL *url.URL
|
||||
RemoteAddr string
|
||||
}
|
||||
|
||||
func NewFetchRequestEnvelope(ctx context.Context, remoteAddr string, url *url.URL) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressFetchRequest, &FetchRequest{
|
||||
Context: ctx,
|
||||
URL: url,
|
||||
RemoteAddr: remoteAddr,
|
||||
})
|
||||
}
|
||||
|
||||
type FetchResponse struct {
|
||||
Allow bool
|
||||
}
|
||||
|
||||
func NewFetchResponseEnvelope(allow bool) bus.Envelope {
|
||||
return bus.NewEnvelope(AddressFetchResponse, &FetchResponse{
|
||||
Allow: allow,
|
||||
})
|
||||
}
|
||||
49
pkg/module/fetch/fetch_message.go
Normal file
49
pkg/module/fetch/fetch_message.go
Normal file
@ -0,0 +1,49 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"github.com/oklog/ulid/v2"
|
||||
)
|
||||
|
||||
const (
|
||||
MessageNamespaceFetchRequest bus.MessageNamespace = "fetchRequest"
|
||||
MessageNamespaceFetchResponse bus.MessageNamespace = "fetchResponse"
|
||||
)
|
||||
|
||||
type MessageFetchRequest struct {
|
||||
Context context.Context
|
||||
RequestID string
|
||||
URL *url.URL
|
||||
RemoteAddr string
|
||||
}
|
||||
|
||||
func (m *MessageFetchRequest) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceFetchRequest
|
||||
}
|
||||
|
||||
func NewMessageFetchRequest(ctx context.Context, remoteAddr string, url *url.URL) *MessageFetchRequest {
|
||||
return &MessageFetchRequest{
|
||||
Context: ctx,
|
||||
RequestID: ulid.Make().String(),
|
||||
RemoteAddr: remoteAddr,
|
||||
URL: url,
|
||||
}
|
||||
}
|
||||
|
||||
type MessageFetchResponse struct {
|
||||
RequestID string
|
||||
Allow bool
|
||||
}
|
||||
|
||||
func (m *MessageFetchResponse) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceFetchResponse
|
||||
}
|
||||
|
||||
func NewMessageFetchResponse(requestID string) *MessageFetchResponse {
|
||||
return &MessageFetchResponse{
|
||||
RequestID: requestID,
|
||||
}
|
||||
}
|
||||
@ -1,127 +0,0 @@
|
||||
package fetch
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
func Mount() func(r chi.Router) {
|
||||
return func(r chi.Router) {
|
||||
r.Get("/api/v1/fetch", handleAppFetch)
|
||||
}
|
||||
}
|
||||
|
||||
func handleAppFetch(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
rawURL := r.URL.Query().Get("url")
|
||||
|
||||
url, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
edgehttp.JSONError(w, http.StatusBadRequest, edgehttp.ErrCodeBadRequest)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
requestMsg := NewFetchRequestEnvelope(ctx, r.RemoteAddr, url)
|
||||
|
||||
bus, ok := edgehttp.ContextBus(ctx)
|
||||
if !ok {
|
||||
logger.Error(ctx, "could find bus on context")
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
reply, err := bus.Request(ctx, requestMsg)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not retrieve fetch request reply", logger.CapturedE(errors.WithStack(err)))
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "fetch reply", logger.F("reply", reply))
|
||||
|
||||
responseMsg, ok := reply.Message().(*FetchResponse)
|
||||
if !ok {
|
||||
logger.Error(
|
||||
ctx, "unexpected fetch response message",
|
||||
logger.F("message", reply),
|
||||
)
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if !responseMsg.Allow {
|
||||
edgehttp.JSONError(w, http.StatusForbidden, edgehttp.ErrCodeForbidden)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
proxyReq, err := http.NewRequest(http.MethodGet, url.String(), nil)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not create proxy request",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
for header, values := range r.Header {
|
||||
for _, value := range values {
|
||||
proxyReq.Header.Add(header, value)
|
||||
}
|
||||
}
|
||||
|
||||
proxyReq.Header.Add("X-Forwarded-From", r.RemoteAddr)
|
||||
|
||||
httpClient, ok := edgehttp.ContextHTTPClient(ctx)
|
||||
if !ok {
|
||||
logger.Error(ctx, "could find http client on context")
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
res, err := httpClient.Do(proxyReq)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not execute proxy request",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
edgehttp.JSONError(w, http.StatusInternalServerError, edgehttp.ErrCodeInternalError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err := res.Body.Close(); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not close response body",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
}
|
||||
}()
|
||||
|
||||
for header, values := range res.Header {
|
||||
for _, value := range values {
|
||||
w.Header().Add(header, value)
|
||||
}
|
||||
}
|
||||
|
||||
w.WriteHeader(res.StatusCode)
|
||||
|
||||
if _, err := io.Copy(w, res.Body); err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
@ -40,15 +40,15 @@ func (m *Module) get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
func (m *Module) handleMessages() {
|
||||
ctx := context.Background()
|
||||
|
||||
fetchErrs := m.bus.Reply(ctx, AddressFetchRequest, func(env bus.Envelope) (any, error) {
|
||||
fetchRequest, ok := env.Message().(*FetchRequest)
|
||||
err := m.bus.Reply(ctx, MessageNamespaceFetchRequest, func(msg bus.Message) (bus.Message, error) {
|
||||
fetchRequest, ok := msg.(*MessageFetchRequest)
|
||||
if !ok {
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected fetch request, got '%T'", env.Message())
|
||||
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message fetch request, got '%T'", msg)
|
||||
}
|
||||
|
||||
res, err := m.handleFetchRequest(fetchRequest)
|
||||
if err != nil {
|
||||
logger.Error(ctx, "could not handle fetch request", logger.CapturedE(errors.WithStack(err)))
|
||||
logger.Error(ctx, "could not handle fetch request", logger.E(errors.WithStack(err)))
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
@ -57,14 +57,13 @@ func (m *Module) handleMessages() {
|
||||
|
||||
return res, nil
|
||||
})
|
||||
|
||||
for err := range fetchErrs {
|
||||
logger.Fatal(ctx, "error while replying to fetch requests", logger.CapturedE(errors.WithStack(err)))
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) handleFetchRequest(req *FetchRequest) (*FetchResponse, error) {
|
||||
res := &FetchResponse{}
|
||||
func (m *Module) handleFetchRequest(req *MessageFetchRequest) (*MessageFetchResponse, error) {
|
||||
res := NewMessageFetchResponse(req.RequestID)
|
||||
|
||||
ctx := logger.With(
|
||||
req.Context,
|
||||
@ -84,11 +83,11 @@ func (m *Module) handleFetchRequest(req *FetchRequest) (*FetchResponse, error) {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
result, ok := rawResult.(map[string]interface{})
|
||||
result, ok := rawResult.Export().(map[string]interface{})
|
||||
if !ok {
|
||||
return nil, errors.Errorf(
|
||||
"unexpected onClientFetch result: expected 'map[string]interface{}', got '%T'",
|
||||
rawResult,
|
||||
rawResult.Export(),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -2,8 +2,8 @@ package fetch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -18,9 +18,7 @@ import (
|
||||
func TestFetchModule(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(slog.LevelDebug)
|
||||
}
|
||||
|
||||
bus := memory.NewBus()
|
||||
|
||||
@ -30,20 +28,21 @@ func TestFetchModule(t *testing.T) {
|
||||
ModuleFactory(bus),
|
||||
)
|
||||
|
||||
path := "testdata/fetch.js"
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
data, err := ioutil.ReadFile("testdata/fetch.js")
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := server.Start(ctx, path, string(data)); err != nil {
|
||||
if err := server.Load("testdata/fetch.js", string(data)); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer server.Stop()
|
||||
|
||||
if err := server.Start(); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
// Wait for module to startup
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
@ -53,33 +52,33 @@ func TestFetchModule(t *testing.T) {
|
||||
remoteAddr := "127.0.0.1"
|
||||
url, _ := url.Parse("http://example.com")
|
||||
|
||||
reply, err := bus.Request(ctx, NewFetchRequestEnvelope(ctx, remoteAddr, url))
|
||||
rawReply, err := bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
response, ok := reply.Message().(*FetchResponse)
|
||||
reply, ok := rawReply.(*MessageFetchResponse)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected reply message type '%T'", reply.Message())
|
||||
t.Fatalf("unexpected reply type '%T'", rawReply)
|
||||
}
|
||||
|
||||
if e, g := true, response.Allow; e != g {
|
||||
if e, g := true, reply.Allow; e != g {
|
||||
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
|
||||
}
|
||||
|
||||
url, _ = url.Parse("https://google.com")
|
||||
|
||||
reply, err = bus.Request(ctx, NewFetchRequestEnvelope(ctx, remoteAddr, url))
|
||||
rawReply, err = bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
response, ok = reply.Message().(*FetchResponse)
|
||||
reply, ok = rawReply.(*MessageFetchResponse)
|
||||
if !ok {
|
||||
t.Fatalf("unexpected reply message type '%T'", reply.Message())
|
||||
t.Fatalf("unexpected reply type '%T'", rawReply)
|
||||
}
|
||||
|
||||
if e, g := false, response.Allow; e != g {
|
||||
if e, g := false, reply.Allow; e != g {
|
||||
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
|
||||
}
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
@ -17,29 +18,30 @@ func (m *LifecycleModule) Name() string {
|
||||
func (m *LifecycleModule) Export(export *goja.Object) {
|
||||
}
|
||||
|
||||
func (m *LifecycleModule) OnInit(ctx context.Context, rt *goja.Runtime) (err error) {
|
||||
func (m *LifecycleModule) OnInit(rt *goja.Runtime) (err error) {
|
||||
call, ok := goja.AssertFunction(rt.Get("onInit"))
|
||||
if !ok {
|
||||
logger.Warn(ctx, "could not find onInit() function")
|
||||
logger.Warn(context.Background(), "could not find onInit() function")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
defer func() {
|
||||
recovered := recover()
|
||||
if recovered == nil {
|
||||
if recovered := recover(); recovered != nil {
|
||||
revoveredErr, ok := recovered.(error)
|
||||
if ok {
|
||||
logger.Error(context.Background(), "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
|
||||
|
||||
err = errors.WithStack(app.ErUnknownError)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
recoveredErr, ok := recovered.(error)
|
||||
if !ok {
|
||||
panic(recovered)
|
||||
}
|
||||
|
||||
err = recoveredErr
|
||||
}()
|
||||
|
||||
call(nil, rt.ToValue(ctx))
|
||||
call(nil)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
38
pkg/module/message.go
Normal file
38
pkg/module/message.go
Normal file
@ -0,0 +1,38 @@
|
||||
package module
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
MessageNamespaceClient bus.MessageNamespace = "client"
|
||||
MessageNamespaceServer bus.MessageNamespace = "server"
|
||||
)
|
||||
|
||||
type ServerMessage struct {
|
||||
Context context.Context
|
||||
Data interface{}
|
||||
}
|
||||
|
||||
func (m *ServerMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceServer
|
||||
}
|
||||
|
||||
func NewServerMessage(ctx context.Context, data interface{}) *ServerMessage {
|
||||
return &ServerMessage{ctx, data}
|
||||
}
|
||||
|
||||
type ClientMessage struct {
|
||||
Context context.Context
|
||||
Data map[string]interface{}
|
||||
}
|
||||
|
||||
func (m *ClientMessage) MessageNamespace() bus.MessageNamespace {
|
||||
return MessageNamespaceClient
|
||||
}
|
||||
|
||||
func NewClientMessage(ctx context.Context, data map[string]interface{}) *ClientMessage {
|
||||
return &ClientMessage{ctx, data}
|
||||
}
|
||||
@ -5,7 +5,8 @@ import (
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
edgeHTTP "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/util"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/pkg/errors"
|
||||
@ -37,9 +38,10 @@ func (m *Module) broadcast(call goja.FunctionCall, rt *goja.Runtime) goja.Value
|
||||
}
|
||||
|
||||
data := call.Argument(0).Export()
|
||||
ctx := context.Background()
|
||||
|
||||
env := edgehttp.NewOutgoingMessageEnvelope("", data)
|
||||
if err := m.bus.Publish(env); err != nil {
|
||||
msg := module.NewServerMessage(ctx, data)
|
||||
if err := m.bus.Publish(ctx, msg); err != nil {
|
||||
panic(rt.ToValue(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
@ -51,36 +53,38 @@ func (m *Module) send(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
panic(rt.ToValue(errors.New("invalid number of argument")))
|
||||
}
|
||||
|
||||
var ctx context.Context
|
||||
|
||||
firstArg := call.Argument(0)
|
||||
|
||||
sessionID, ok := firstArg.Export().(string)
|
||||
if !ok {
|
||||
ctx := util.AssertContext(firstArg, rt)
|
||||
sessionID, ok = edgehttp.ContextSessionID(ctx)
|
||||
if !ok {
|
||||
panic(rt.ToValue(errors.New("could not find session id in context")))
|
||||
}
|
||||
if ok {
|
||||
ctx = module.WithContext(context.Background(), map[module.ContextKey]any{
|
||||
edgeHTTP.ContextKeySessionID: sessionID,
|
||||
})
|
||||
} else {
|
||||
ctx = util.AssertContext(firstArg, rt)
|
||||
}
|
||||
|
||||
data := call.Argument(1).Export()
|
||||
|
||||
env := edgehttp.NewOutgoingMessageEnvelope(sessionID, data)
|
||||
if err := m.bus.Publish(env); err != nil {
|
||||
msg := module.NewServerMessage(ctx, data)
|
||||
if err := m.bus.Publish(ctx, msg); err != nil {
|
||||
panic(rt.ToValue(errors.WithStack(err)))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) handleIncomingMessages() {
|
||||
func (m *Module) handleClientMessages() {
|
||||
ctx := context.Background()
|
||||
|
||||
logger.Debug(
|
||||
ctx,
|
||||
"subscribing to bus envelopes",
|
||||
"subscribing to bus messages",
|
||||
)
|
||||
|
||||
envelopes, err := m.bus.Subscribe(ctx, edgehttp.AddressIncomingMessage)
|
||||
clientMessages, err := m.bus.Subscribe(ctx, module.MessageNamespaceClient)
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
@ -88,16 +92,16 @@ func (m *Module) handleIncomingMessages() {
|
||||
defer func() {
|
||||
logger.Debug(
|
||||
ctx,
|
||||
"unsubscribing from bus envelopes",
|
||||
"unsubscribing from bus messages",
|
||||
)
|
||||
|
||||
m.bus.Unsubscribe(edgehttp.AddressIncomingMessage, envelopes)
|
||||
m.bus.Unsubscribe(ctx, module.MessageNamespaceClient, clientMessages)
|
||||
}()
|
||||
|
||||
for {
|
||||
logger.Debug(
|
||||
ctx,
|
||||
"waiting for next envelope",
|
||||
"waiting for next message",
|
||||
)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@ -108,13 +112,13 @@ func (m *Module) handleIncomingMessages() {
|
||||
|
||||
return
|
||||
|
||||
case env := <-envelopes:
|
||||
incomingMessage, ok := env.Message().(*edgehttp.IncomingMessage)
|
||||
case msg := <-clientMessages:
|
||||
clientMessage, ok := msg.(*module.ClientMessage)
|
||||
if !ok {
|
||||
logger.Warn(
|
||||
logger.Error(
|
||||
ctx,
|
||||
"unexpected message type",
|
||||
logger.F("message", env.Message()),
|
||||
logger.F("message", msg),
|
||||
)
|
||||
|
||||
continue
|
||||
@ -122,11 +126,11 @@ func (m *Module) handleIncomingMessages() {
|
||||
|
||||
logger.Debug(
|
||||
ctx,
|
||||
"received incoming message",
|
||||
logger.F("message", incomingMessage),
|
||||
"received client message",
|
||||
logger.F("message", clientMessage),
|
||||
)
|
||||
|
||||
if _, err := m.server.ExecFuncByName(incomingMessage.Context, "onClientMessage", incomingMessage.Context, incomingMessage.Payload); err != nil {
|
||||
if _, err := m.server.ExecFuncByName(clientMessage.Context, "onClientMessage", clientMessage.Context, clientMessage.Data); err != nil {
|
||||
if errors.Is(err, app.ErrFuncDoesNotExist) {
|
||||
continue
|
||||
}
|
||||
@ -134,7 +138,7 @@ func (m *Module) handleIncomingMessages() {
|
||||
logger.Error(
|
||||
ctx,
|
||||
"on client message error",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
logger.E(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -148,7 +152,7 @@ func ModuleFactory(bus bus.Bus) app.ServerModuleFactory {
|
||||
bus: bus,
|
||||
}
|
||||
|
||||
go module.handleIncomingMessages()
|
||||
go module.handleClientMessages()
|
||||
|
||||
return module
|
||||
}
|
||||
|
||||
280
pkg/module/rpc.go
Normal file
280
pkg/module/rpc.go
Normal file
@ -0,0 +1,280 @@
|
||||
package module
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/util"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type RPCRequest struct {
|
||||
Method string
|
||||
Params interface{}
|
||||
ID interface{}
|
||||
}
|
||||
|
||||
type RPCError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data interface{} `json:"data"`
|
||||
}
|
||||
|
||||
type RPCResponse struct {
|
||||
Result interface{}
|
||||
Error *RPCError
|
||||
ID interface{}
|
||||
}
|
||||
|
||||
type RPCModule struct {
|
||||
server *app.Server
|
||||
bus bus.Bus
|
||||
callbacks sync.Map
|
||||
}
|
||||
|
||||
func (m *RPCModule) Name() string {
|
||||
return "rpc"
|
||||
}
|
||||
|
||||
func (m *RPCModule) Export(export *goja.Object) {
|
||||
if err := export.Set("register", m.register); err != nil {
|
||||
panic(errors.Wrap(err, "could not set 'register' function"))
|
||||
}
|
||||
|
||||
if err := export.Set("unregister", m.unregister); err != nil {
|
||||
panic(errors.Wrap(err, "could not set 'unregister' function"))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *RPCModule) OnInit(rt *goja.Runtime) error {
|
||||
go m.handleMessages()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RPCModule) register(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
fnName := util.AssertString(call.Argument(0), rt)
|
||||
|
||||
var (
|
||||
callable goja.Callable
|
||||
ok bool
|
||||
)
|
||||
|
||||
if len(call.Arguments) > 1 {
|
||||
callable, ok = goja.AssertFunction(call.Argument(1))
|
||||
} else {
|
||||
callable, ok = goja.AssertFunction(rt.Get(fnName))
|
||||
}
|
||||
|
||||
if !ok {
|
||||
panic(rt.NewTypeError("method should be a valid function"))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
logger.Debug(ctx, "registering method", logger.F("method", fnName))
|
||||
|
||||
m.callbacks.Store(fnName, callable)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RPCModule) unregister(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
fnName := util.AssertString(call.Argument(0), rt)
|
||||
|
||||
m.callbacks.Delete(fnName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RPCModule) handleMessages() {
|
||||
ctx := context.Background()
|
||||
|
||||
clientMessages, err := m.bus.Subscribe(ctx, MessageNamespaceClient)
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer func() {
|
||||
m.bus.Unsubscribe(ctx, MessageNamespaceClient, clientMessages)
|
||||
}()
|
||||
|
||||
sendRes := func(ctx context.Context, req *RPCRequest, result goja.Value) {
|
||||
res := &RPCResponse{
|
||||
ID: req.ID,
|
||||
Result: result.Export(),
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "sending rpc response", logger.F("response", res))
|
||||
|
||||
if err := m.sendResponse(ctx, res); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send response",
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("response", res),
|
||||
logger.F("request", req),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
for msg := range clientMessages {
|
||||
go m.handleMessage(ctx, msg, sendRes)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *RPCModule) handleMessage(ctx context.Context, msg bus.Message, sendRes func(ctx context.Context, req *RPCRequest, result goja.Value)) {
|
||||
clientMessage, ok := msg.(*ClientMessage)
|
||||
if !ok {
|
||||
logger.Warn(ctx, "unexpected bus message", logger.F("message", msg))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
ok, req := m.isRPCRequest(clientMessage)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
logger.Debug(ctx, "received rpc request", logger.F("request", req))
|
||||
|
||||
rawCallable, exists := m.callbacks.Load(req.Method)
|
||||
if !exists {
|
||||
logger.Debug(ctx, "method not found", logger.F("req", req))
|
||||
|
||||
if err := m.sendMethodNotFoundResponse(clientMessage.Context, req); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send method not found response",
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("request", req),
|
||||
)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
callable, ok := rawCallable.(goja.Callable)
|
||||
if !ok {
|
||||
logger.Debug(ctx, "invalid method", logger.F("req", req))
|
||||
|
||||
if err := m.sendMethodNotFoundResponse(clientMessage.Context, req); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send method not found response",
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("request", req),
|
||||
)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
result, err := m.server.Exec(clientMessage.Context, callable, clientMessage.Context, req.Params)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "rpc call error",
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("request", req),
|
||||
)
|
||||
|
||||
if err := m.sendErrorResponse(clientMessage.Context, req, err); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send error response",
|
||||
logger.E(errors.WithStack(err)),
|
||||
logger.F("originalError", err),
|
||||
logger.F("request", req),
|
||||
)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
promise, ok := app.IsPromise(result)
|
||||
if ok {
|
||||
go func(ctx context.Context, req *RPCRequest, promise *goja.Promise) {
|
||||
result := m.server.WaitForPromise(promise)
|
||||
sendRes(ctx, req, result)
|
||||
}(clientMessage.Context, req, promise)
|
||||
} else {
|
||||
sendRes(clientMessage.Context, req, result)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *RPCModule) sendErrorResponse(ctx context.Context, req *RPCRequest, err error) error {
|
||||
return m.sendResponse(ctx, &RPCResponse{
|
||||
ID: req.ID,
|
||||
Result: nil,
|
||||
Error: &RPCError{
|
||||
Code: -32603,
|
||||
Message: err.Error(),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (m *RPCModule) sendMethodNotFoundResponse(ctx context.Context, req *RPCRequest) error {
|
||||
return m.sendResponse(ctx, &RPCResponse{
|
||||
ID: req.ID,
|
||||
Result: nil,
|
||||
Error: &RPCError{
|
||||
Code: -32601,
|
||||
Message: fmt.Sprintf("method not found"),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func (m *RPCModule) sendResponse(ctx context.Context, res *RPCResponse) error {
|
||||
msg := NewServerMessage(ctx, map[string]interface{}{
|
||||
"jsonrpc": "2.0",
|
||||
"id": res.ID,
|
||||
"error": res.Error,
|
||||
"result": res.Result,
|
||||
})
|
||||
|
||||
if err := m.bus.Publish(ctx, msg); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *RPCModule) isRPCRequest(msg *ClientMessage) (bool, *RPCRequest) {
|
||||
jsonRPC, exists := msg.Data["jsonrpc"]
|
||||
if !exists || jsonRPC != "2.0" {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
rawMethod, exists := msg.Data["method"]
|
||||
if !exists {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
method, ok := rawMethod.(string)
|
||||
if !ok {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
id := msg.Data["id"]
|
||||
params := msg.Data["params"]
|
||||
|
||||
return true, &RPCRequest{
|
||||
ID: id,
|
||||
Method: method,
|
||||
Params: params,
|
||||
}
|
||||
}
|
||||
|
||||
func RPCModuleFactory(bus bus.Bus) app.ServerModuleFactory {
|
||||
return func(server *app.Server) app.ServerModule {
|
||||
mod := &RPCModule{
|
||||
server: server,
|
||||
bus: bus,
|
||||
}
|
||||
|
||||
return mod
|
||||
}
|
||||
}
|
||||
|
||||
var _ app.InitializableModule = &RPCModule{}
|
||||
@ -1,21 +0,0 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
)
|
||||
|
||||
const (
|
||||
Address bus.Address = "module/rpc"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
Context context.Context
|
||||
Method string
|
||||
Params any
|
||||
}
|
||||
|
||||
func NewRequestEnvelope(ctx context.Context, method string, params any) bus.Envelope {
|
||||
return bus.NewEnvelope(Address, &Request{ctx, method, params})
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
package rpc
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrMethodNotFound = errors.New("method not found")
|
||||
)
|
||||
@ -1,19 +0,0 @@
|
||||
package rpc
|
||||
|
||||
import "fmt"
|
||||
|
||||
type JSONRPCRequest struct {
|
||||
ID any
|
||||
Method string
|
||||
Params any
|
||||
}
|
||||
|
||||
type JSONRPCError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data interface{} `json:"data"`
|
||||
}
|
||||
|
||||
func (e *JSONRPCError) Error() string {
|
||||
return fmt.Sprintf("json-rpc error: %d - %s", e.Code, e.Message)
|
||||
}
|
||||
@ -1,260 +0,0 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
edgehttp "forge.cadoles.com/arcad/edge/pkg/http"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module/util"
|
||||
"github.com/dop251/goja"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
type Module struct {
|
||||
server *app.Server
|
||||
bus bus.Bus
|
||||
callbacks sync.Map
|
||||
}
|
||||
|
||||
func (m *Module) Name() string {
|
||||
return "rpc"
|
||||
}
|
||||
|
||||
func (m *Module) Export(export *goja.Object) {
|
||||
if err := export.Set("register", m.register); err != nil {
|
||||
panic(errors.Wrap(err, "could not set 'register' function"))
|
||||
}
|
||||
|
||||
if err := export.Set("unregister", m.unregister); err != nil {
|
||||
panic(errors.Wrap(err, "could not set 'unregister' function"))
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) OnInit(ctx context.Context, rt *goja.Runtime) error {
|
||||
requestErrs := m.bus.Reply(ctx, Address, m.handleRequest)
|
||||
go func() {
|
||||
for err := range requestErrs {
|
||||
logger.Error(ctx, "error while replying to rpc requests", logger.CapturedE(errors.WithStack(err)))
|
||||
}
|
||||
}()
|
||||
|
||||
httpIncomingMessages, err := m.bus.Subscribe(ctx, edgehttp.AddressIncomingMessage)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
go m.handleIncomingHTTPMessages(ctx, httpIncomingMessages)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) register(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
fnName := util.AssertString(call.Argument(0), rt)
|
||||
|
||||
var (
|
||||
callable goja.Callable
|
||||
ok bool
|
||||
)
|
||||
|
||||
if len(call.Arguments) > 1 {
|
||||
callable, ok = goja.AssertFunction(call.Argument(1))
|
||||
} else {
|
||||
callable, ok = goja.AssertFunction(rt.Get(fnName))
|
||||
}
|
||||
|
||||
if !ok {
|
||||
panic(rt.NewTypeError("method should be a valid function"))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
logger.Debug(ctx, "registering method", logger.F("method", fnName))
|
||||
|
||||
m.callbacks.Store(fnName, callable)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) unregister(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
|
||||
fnName := util.AssertString(call.Argument(0), rt)
|
||||
|
||||
m.callbacks.Delete(fnName)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) handleRequest(env bus.Envelope) (any, error) {
|
||||
request, ok := env.Message().(*Request)
|
||||
if !ok {
|
||||
logger.Warn(context.Background(), "unexpected bus message", logger.F("message", env.Message()))
|
||||
|
||||
return nil, errors.WithStack(bus.ErrUnexpectedMessage)
|
||||
}
|
||||
|
||||
ctx := logger.With(request.Context, logger.F("request", request))
|
||||
|
||||
logger.Debug(ctx, "received rpc request")
|
||||
|
||||
rawCallable, exists := m.callbacks.Load(request.Method)
|
||||
if !exists {
|
||||
logger.Debug(ctx, "method not found")
|
||||
|
||||
return nil, errors.WithStack(ErrMethodNotFound)
|
||||
}
|
||||
|
||||
callable, ok := rawCallable.(goja.Callable)
|
||||
if !ok {
|
||||
logger.Debug(ctx, "invalid method")
|
||||
|
||||
return nil, errors.WithStack(ErrMethodNotFound)
|
||||
}
|
||||
|
||||
result, err := m.server.Exec(ctx, callable, request.Context, request.Params)
|
||||
if err != nil {
|
||||
logger.Error(
|
||||
ctx, "rpc call error",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m *Module) handleIncomingHTTPMessages(ctx context.Context, incoming <-chan bus.Envelope) {
|
||||
defer func() {
|
||||
m.bus.Unsubscribe(edgehttp.AddressIncomingMessage, incoming)
|
||||
}()
|
||||
|
||||
for env := range incoming {
|
||||
msg, ok := env.Message().(*edgehttp.IncomingMessage)
|
||||
if !ok {
|
||||
logger.Error(ctx, "unexpected incoming http message type", logger.F("message", env.Message()))
|
||||
continue
|
||||
}
|
||||
|
||||
jsonReq, ok := m.isRPCRequest(msg.Payload)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
sessionID, ok := edgehttp.ContextSessionID(msg.Context)
|
||||
if !ok {
|
||||
logger.Error(ctx, "could not find session id in context")
|
||||
continue
|
||||
}
|
||||
|
||||
request := NewRequestEnvelope(msg.Context, jsonReq.Method, jsonReq.Params)
|
||||
|
||||
requestCtx := logger.With(msg.Context, logger.F("rpcRequestMethod", jsonReq.Method), logger.F("rpcRequestID", jsonReq.ID))
|
||||
|
||||
reply, err := m.bus.Request(requestCtx, request)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
|
||||
logger.Error(
|
||||
ctx, "could not execute rpc request",
|
||||
logger.CapturedE(err),
|
||||
)
|
||||
|
||||
if errors.Is(err, ErrMethodNotFound) {
|
||||
if err := m.sendMethodNotFoundResponse(sessionID, jsonReq.ID); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send json rpc error response",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := m.sendErrorResponse(sessionID, jsonReq.ID, err); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send json rpc error response",
|
||||
logger.CapturedE(errors.WithStack(err)),
|
||||
)
|
||||
}
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
if err := m.sendResponse(sessionID, jsonReq.ID, reply.Message(), nil); err != nil {
|
||||
logger.Error(
|
||||
ctx, "could not send json rpc result response",
|
||||
logger.CapturedE(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) sendErrorResponse(sessionID string, requestID any, err error) error {
|
||||
return m.sendResponse(sessionID, requestID, nil, &JSONRPCError{
|
||||
Code: -32603,
|
||||
Message: err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Module) sendMethodNotFoundResponse(sessionID string, requestID any) error {
|
||||
return m.sendResponse(sessionID, requestID, nil, &JSONRPCError{
|
||||
Code: -32601,
|
||||
Message: "method not found",
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Module) sendResponse(sessionID string, requestID any, result any, err error) error {
|
||||
env := edgehttp.NewOutgoingMessageEnvelope(sessionID, map[string]interface{}{
|
||||
"jsonrpc": "2.0",
|
||||
"id": requestID,
|
||||
"error": err,
|
||||
"result": result,
|
||||
})
|
||||
|
||||
if err := m.bus.Publish(env); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Module) isRPCRequest(payload map[string]any) (*JSONRPCRequest, bool) {
|
||||
jsonRPC, exists := payload["jsonrpc"]
|
||||
if !exists || jsonRPC != "2.0" {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
rawMethod, exists := payload["method"]
|
||||
if !exists {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
method, ok := rawMethod.(string)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
id := payload["id"]
|
||||
params := payload["params"]
|
||||
|
||||
return &JSONRPCRequest{
|
||||
ID: id,
|
||||
Method: method,
|
||||
Params: params,
|
||||
}, true
|
||||
}
|
||||
|
||||
func ModuleFactory(bus bus.Bus) app.ServerModuleFactory {
|
||||
return func(server *app.Server) app.ServerModule {
|
||||
mod := &Module{
|
||||
server: server,
|
||||
bus: bus,
|
||||
}
|
||||
|
||||
return mod
|
||||
}
|
||||
}
|
||||
|
||||
var _ app.InitializableModule = &Module{}
|
||||
@ -1,109 +0,0 @@
|
||||
package rpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"forge.cadoles.com/arcad/edge/pkg/app"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus"
|
||||
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
|
||||
"forge.cadoles.com/arcad/edge/pkg/module"
|
||||
"github.com/pkg/errors"
|
||||
"gitlab.com/wpetit/goweb/logger"
|
||||
)
|
||||
|
||||
func TestServerExecDeadlock(t *testing.T) {
|
||||
if testing.Verbose() {
|
||||
logger.SetLevel(logger.LevelDebug)
|
||||
}
|
||||
|
||||
b := memory.NewBus(memory.WithBufferSize(1))
|
||||
|
||||
server := app.NewServer(
|
||||
module.ConsoleModuleFactory(),
|
||||
ModuleFactory(b),
|
||||
module.LifecycleModuleFactory(),
|
||||
)
|
||||
|
||||
data, err := os.ReadFile("testdata/deadlock.js")
|
||||
if err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
t.Log("starting server")
|
||||
|
||||
if err := server.Start(ctx, "deadlock.js", string(data)); err != nil {
|
||||
t.Fatalf("%+v", errors.WithStack(err))
|
||||
}
|
||||
|
||||
defer server.Stop()
|
||||
|
||||
t.Log("server started")
|
||||
|
||||
count := 100
|
||||
delay := 100
|
||||
|
||||
var wg sync.WaitGroup
|
||||
|
||||
wg.Add(count)
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
|
||||
t.Logf("calling %d", i)
|
||||
|
||||
isCanceled := i%2 == 0
|
||||
|
||||
var ctx context.Context
|
||||
if isCanceled {
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
ctx = canceledCtx
|
||||
} else {
|
||||
ctx = context.Background()
|
||||
}
|
||||
|
||||
env := NewRequestEnvelope(ctx, "doSomethingLong", map[string]any{
|
||||
"i": i,
|
||||
"delay": delay,
|
||||
})
|
||||
|
||||
t.Logf("publishing envelope #%d", i)
|
||||
|
||||
reply, err := b.Request(ctx, env)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) && isCanceled {
|
||||
return
|
||||
}
|
||||
|
||||
if errors.Is(err, bus.ErrNoResponse) && isCanceled {
|
||||
return
|
||||
}
|
||||
|
||||
t.Errorf("%+v", errors.WithStack(err))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
result, ok := reply.Message().(int64)
|
||||
if !ok {
|
||||
t.Errorf("response.Result: expected type '%T', got '%T'", int64(0), reply.Message())
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
if e, g := i, int(result); e != g {
|
||||
t.Errorf("response.Result: expected '%v', got '%v'", e, g)
|
||||
|
||||
return
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
14
pkg/module/rpc/testdata/deadlock.js
vendored
14
pkg/module/rpc/testdata/deadlock.js
vendored
@ -1,14 +0,0 @@
|
||||
function onInit() {
|
||||
rpc.register("doSomethingLong", doSomethingLong)
|
||||
}
|
||||
|
||||
function doSomethingLong(ctx, params) {
|
||||
var start = Date.now()
|
||||
|
||||
while (true) {
|
||||
var now = Date.now()
|
||||
if (now - start >= params.delay) break
|
||||
}
|
||||
|
||||
return params.i;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user