Compare commits

...

31 Commits

Author SHA1 Message Date
310dac296f feat(storage,sqlite): begin tx with context
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-13 11:23:34 +02:00
4db7576b12 feat(client,sdk): retrieve auth token from parent frame + better resize detection
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-13 11:02:24 +02:00
f5283b86ed fix(app,manifest): manifest serialization
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-11 15:08:07 +02:00
98ebd7a168 doc(app,manifest): add metadata attribute in manifest schema
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-11 11:11:00 +02:00
8ca31d05c0 feat(app,manifest): validation + extendable metadatas
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-11 11:05:09 +02:00
34c6a089b5 fix(client,sdk): permit cross-domain message communication
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 20:54:01 +02:00
da73b842e1 fix(sdk,client): initialize crossframe observers after window load event
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 19:18:36 +02:00
55d7241d95 chore(sdk,client): remove restrictive assertion
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 18:18:12 +02:00
240b07af66 feat(sdk,client): add edgeframe sdk api
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 18:16:17 +02:00
68e35bf5a6 fix(client,sdk): remove too specific assertion
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 15:59:09 +02:00
4bc2d864ad chore: add jenkins ci pipeline
All checks were successful
arcad/edge/pipeline/head This commit looks good
2023-04-06 14:58:12 +02:00
dc18381dea chore: add test timeout 2023-04-06 14:47:37 +02:00
1dde96043a chore: reenable tests in watch mode 2023-04-06 14:47:13 +02:00
f758acb4e5 fix(module,fetch): wait for module initialization to prevent false failure in test 2023-04-06 14:46:46 +02:00
054e80bbfb fix(storage,sqlite): prevent 'database is busy' error by using busy_timeout pragma 2023-04-06 14:45:50 +02:00
32c6f0a77e feat(cli,run): resolve app url based on available network interfaces 2023-04-06 11:52:04 +02:00
050e529f0a doc(module,app): add new parameter 'from' to app.getUrl() 2023-04-06 11:27:27 +02:00
006f13bc7b feat(module,auth): dynamically define authentication cookie domain 2023-04-05 15:19:22 +02:00
84c8fd51f6 chore: add cast commands for testing purpose 2023-04-05 15:12:51 +02:00
f08f645432 chore: fix gitea-release task 2023-04-02 18:01:47 +02:00
fbb27d6ea4 feat(app,module): fetch basic module 2023-04-02 17:59:33 +02:00
d8ce2901d2 feat(jwt): handle nil keyset 2023-03-28 20:38:29 +02:00
1996f4dc56 feat(auth,local_handler): cookie configuration 2023-03-28 20:37:57 +02:00
e09de0b0a4 chore: move proxy package to arcad/emissary 2023-03-28 10:15:49 +02:00
72765de20b fix(doc): typo 2023-03-24 11:05:17 +01:00
ed535b6f5d feat(module,app): basic module to list apps 2023-03-24 10:59:15 +01:00
07452ad8ab chore: automatically update sdk-testsuite app version 2023-03-23 19:13:47 +01:00
0f0fdfb02b chore: use date based versioning 2023-03-23 19:04:29 +01:00
9eefce9b41 feat: remove arbitrary timeout 2023-03-23 19:01:48 +01:00
0577762be9 feat(module,blob): implements full api 2023-03-23 19:01:20 +01:00
cf8a3f8ac0 feat: add proxy package 2023-03-22 18:05:44 +01:00
84 changed files with 2860 additions and 617 deletions

49
Jenkinsfile vendored Normal file
View File

@ -0,0 +1,49 @@
@Library('cadoles') _
pipeline {
agent {
dockerfile {
label 'docker'
filename 'Dockerfile'
dir 'misc/jenkins'
}
}
stages {
stage('Run unit tests') {
steps {
script {
sh 'make GOTEST_ARGS="-timeout 10m -count=1 -v" test'
}
}
}
stage('Release') {
when {
anyOf {
branch 'master'
branch 'develop'
}
}
steps {
script {
withCredentials([
usernamePassword([
credentialsId: 'forge-jenkins',
usernameVariable: 'GITEA_RELEASE_USERNAME',
passwordVariable: 'GITEA_RELEASE_PASSWORD'
])
]) {
sh 'make gitea-release'
}
}
}
}
}
post {
always {
cleanWs()
}
}
}

View File

@ -2,13 +2,15 @@ LINT_ARGS ?= --timeout 5m
GITCHLOG_ARGS ?=
SHELL := /bin/bash
GOTEST_ARGS ?= -short
GOTEST_ARGS ?= -short -timeout 60s
ESBUILD_VERSION ?= v0.17.5
GIT_VERSION := $(shell git describe --always)
DATE_VERSION := $(shell date +%Y.%-m.%-d)
FULL_VERSION := v$(DATE_VERSION)-$(GIT_VERSION)$(if $(shell git diff --stat),-dirty,)
build: build-edge-cli
build: build-edge-cli build-client-sdk-test-app
watch:
go run -mod=readonly github.com/cortesi/modd/cmd/modd@latest
@ -28,10 +30,12 @@ build-edge-cli: build-sdk
-o ./bin/cli \
./cmd/cli
build-client-sdk-test-app:
cd misc/client-sdk-testsuite && $(MAKE) dist
install-git-hooks:
git config core.hooksPath .githooks
tools/esbuild/bin/esbuild:
mkdir -p tools/esbuild/bin
curl -fsSL https://esbuild.github.io/dl/$(ESBUILD_VERSION) | sh
@ -51,34 +55,40 @@ pkg/sdk/client/dist/client.js: tools/esbuild/bin/esbuild node_modules
--global-name=Edge \
--define:global=window \
--platform=browser \
--footer:js="Edge=Edge.default;" \
--footer:js="EdgeFrame=Edge.crossFrameMessenger;Edge=Edge.client" \
--outfile=pkg/sdk/client/dist/client.js
node_modules:
npm ci
gitea-release: tools/gitea-release/bin/gitea-release.sh build
gitea-release: tools/yq/bin/yq tools/gitea-release/bin/gitea-release.sh build
mkdir -p .gitea-release
rm -rf .gitea-release/*
cp bin/cli .gitea-release/edge_cli_amd64
# Create client-sdk-testsuite package
tools/yq/bin/yq -i '.version = "$(FULL_VERSION)"' ./misc/client-sdk-testsuite/dist/manifest.yml
.gitea-release/edge_cli_amd64 app package -d ./misc/client-sdk-testsuite/dist -o .gitea-release
GITEA_RELEASE_PROJECT="edge" \
GITEA_RELEASE_ORG="arcad" \
GITEA_RELEASE_BASE_URL="https://forge.cadoles.com" \
GITEA_RELEASE_VERSION="$(GIT_VERSION)" \
GITEA_RELEASE_NAME="$(GIT_VERSION)" \
GITEA_RELEASE_VERSION="$(FULL_VERSION)" \
GITEA_RELEASE_NAME="$(FULL_VERSION)" \
GITEA_RELEASE_COMMITISH_TARGET="$(GIT_VERSION)" \
GITEA_RELEASE_IS_DRAFT="false" \
GITEA_RELEASE_IS_PRERELEASE="true" \
GITEA_RELEASE_BODY="" \
GITEA_RELEASE_ATTACHMENTS="$(shell find .gitea-release/* -type f)" \
GITEA_RELEASE_ATTACHMENTS="$$(find .gitea-release/* -type f)" \
tools/gitea-release/bin/gitea-release.sh
tools/gitea-release/bin/gitea-release.sh:
mkdir -p tools/gitea-release/bin
curl --output tools/gitea-release/bin/gitea-release.sh https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/gitea/gitea-release.sh
chmod +x tools/gitea-release/bin/gitea-release.sh
chmod +x tools/gitea-release/bin/gitea-release.sh
tools/yq/bin/yq:
mkdir -p tools/yq/bin
curl -L --output tools/yq/bin/yq https://github.com/mikefarah/yq/releases/download/v4.31.1/yq_linux_amd64
chmod +x tools/yq/bin/yq

View File

@ -0,0 +1,11 @@
package app
import (
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/app/metadata"
)
var manifestMetadataValidators = []app.MetadataValidator{
metadata.WithMinimumRoleValidator("visitor", "user", "superuser", "admin", "superadmin"),
metadata.WithNamedPathsValidator(metadata.NamedPathAdmin, metadata.NamedPathIcon),
}

View File

@ -52,6 +52,10 @@ func PackageCommand() *cli.Command {
return errors.Wrap(err, "could not load app manifest")
}
if valid, err := manifest.Validate(manifestMetadataValidators...); !valid {
return errors.Wrap(err, "invalid app manifest")
}
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return errors.Wrapf(err, "could not create directory ''%s'", outputDir)
}

View File

@ -1,8 +1,11 @@
package app
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"path/filepath"
@ -13,10 +16,14 @@ import (
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
appHTTP "forge.cadoles.com/arcad/edge/pkg/http"
"forge.cadoles.com/arcad/edge/pkg/module"
appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
appModuleMemory "forge.cadoles.com/arcad/edge/pkg/module/app/memory"
"forge.cadoles.com/arcad/edge/pkg/module/auth"
authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
"forge.cadoles.com/arcad/edge/pkg/module/blob"
"forge.cadoles.com/arcad/edge/pkg/module/cast"
"forge.cadoles.com/arcad/edge/pkg/module/net"
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
netModule "forge.cadoles.com/arcad/edge/pkg/module/net"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"gitlab.com/wpetit/goweb/logger"
@ -66,7 +73,7 @@ func RunCommand() *cli.Command {
&cli.StringFlag{
Name: "storage-file",
Usage: "use `FILE` for SQLite storage database",
Value: ".edge/%APPID%/data.sqlite",
Value: ".edge/%APPID%/data.sqlite?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000",
},
&cli.StringFlag{
Name: "accounts-file",
@ -103,8 +110,16 @@ func RunCommand() *cli.Command {
return errors.Wrap(err, "could not load manifest from app bundle")
}
if valid, err := manifest.Validate(manifestMetadataValidators...); !valid {
return errors.Wrap(err, "invalid app manifest")
}
storageFile := injectAppID(ctx.String("storage-file"), manifest.ID)
if err := ensureDir(storageFile); err != nil {
return errors.WithStack(err)
}
db, err := sqlite.Open(storageFile)
if err != nil {
return errors.WithStack(err)
@ -116,7 +131,7 @@ func RunCommand() *cli.Command {
handler := appHTTP.NewHandler(
appHTTP.WithBus(bus),
appHTTP.WithServerModules(getServerModules(bus, ds, bs)...),
appHTTP.WithServerModules(getServerModules(bus, ds, bs, manifest, address)...),
)
if err := handler.Load(bundle); err != nil {
return errors.Wrap(err, "could not load app bundle")
@ -157,16 +172,16 @@ func RunCommand() *cli.Command {
}
}
func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStore) []app.ServerModuleFactory {
func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStore, manifest *app.Manifest, address string) []app.ServerModuleFactory {
return []app.ServerModuleFactory{
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
cast.CastModuleFactory(),
module.LifecycleModuleFactory(),
net.ModuleFactory(bus),
netModule.ModuleFactory(bus),
module.RPCModuleFactory(bus),
module.StoreModuleFactory(ds),
module.BlobModuleFactory(bus, bs),
blob.ModuleFactory(bus, bs),
module.Extends(
auth.ModuleFactory(
auth.WithJWT(dummyKeySet),
@ -189,6 +204,28 @@ func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStor
}
},
),
appModule.ModuleFactory(appModuleMemory.NewRepository(
func(ctx context.Context, id app.ID, from string) (string, error) {
addr := address
if strings.HasPrefix(addr, ":") {
addr = "0.0.0.0" + addr
}
host, port, err := net.SplitHostPort(addr)
if err != nil {
return "", errors.WithStack(err)
}
addr, err = findMatchingDeviceAddress(ctx, from, host)
if err != nil {
return "", errors.WithStack(err)
}
return fmt.Sprintf("http://%s:%s", addr, port), nil
},
manifest,
)),
fetch.ModuleFactory(bus),
}
}
@ -263,3 +300,52 @@ func loadLocalAccounts(path string) ([]authHTTP.LocalAccount, error) {
return accounts, nil
}
func findMatchingDeviceAddress(ctx context.Context, from string, defaultAddr string) (string, error) {
if from == "" {
return defaultAddr, nil
}
fromIP := net.ParseIP(from)
if fromIP == nil {
return defaultAddr, nil
}
ifaces, err := net.Interfaces()
if err != nil {
return "", errors.WithStack(err)
}
for _, ifa := range ifaces {
addrs, err := ifa.Addrs()
if err != nil {
logger.Error(
ctx, "could not retrieve iface adresses",
logger.E(errors.WithStack(err)), logger.F("iface", ifa.Name),
)
continue
}
for _, addr := range addrs {
ip, network, err := net.ParseCIDR(addr.String())
if err != nil {
logger.Error(
ctx, "could not parse address",
logger.E(errors.WithStack(err)), logger.F("address", addr.String()),
)
continue
}
if !network.Contains(fromIP) {
continue
}
return ip.String(), nil
}
}
return defaultAddr, nil
}

View File

@ -0,0 +1,40 @@
package cast
import (
"forge.cadoles.com/arcad/edge/pkg/module/cast"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
)
func LoadURLCommand() *cli.Command {
return &cli.Command{
Name: "load-url",
Usage: "Load `URL` in casting device",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "device",
Aliases: []string{"d"},
Required: true,
},
&cli.StringFlag{
Name: "url",
Aliases: []string{"u"},
Required: true,
},
},
Action: func(ctx *cli.Context) error {
device := ctx.String("device")
url := ctx.String("url")
if err := cast.StopCast(ctx.Context, device); err != nil {
return errors.WithStack(err)
}
if err := cast.LoadURL(ctx.Context, device, url); err != nil {
return errors.WithStack(err)
}
return nil
},
}
}

View File

@ -0,0 +1,16 @@
package cast
import (
"github.com/urfave/cli/v2"
)
func Root() *cli.Command {
return &cli.Command{
Name: "cast",
Usage: "Cast related commands",
Subcommands: []*cli.Command{
ScanCommand(),
LoadURLCommand(),
},
}
}

View File

@ -0,0 +1,37 @@
package cast
import (
"context"
"log"
"time"
"github.com/barnybug/go-cast/discovery"
"github.com/pkg/errors"
"github.com/urfave/cli/v2"
)
func ScanCommand() *cli.Command {
return &cli.Command{
Name: "scan",
Usage: "Scan network for casting devices",
Flags: []cli.Flag{},
Action: func(ctx *cli.Context) error {
service := discovery.NewService(ctx.Context)
defer service.Stop()
go func() {
if err := service.Run(ctx.Context, time.Second); err != nil && !errors.Is(err, context.DeadlineExceeded) {
log.Fatalf("%+v", errors.WithStack(err))
}
}()
found := service.Found()
for device := range found {
log.Printf("[DEVICE] %s %s %s:%d", device.Uuid(), device.Name(), device.IP().String(), device.Port())
}
return nil
},
}
}

View File

@ -3,8 +3,9 @@ package main
import (
"forge.cadoles.com/arcad/edge/cmd/cli/command"
"forge.cadoles.com/arcad/edge/cmd/cli/command/app"
"forge.cadoles.com/arcad/edge/cmd/cli/command/cast"
)
func main() {
command.Main(app.Root())
command.Main(app.Root(), cast.Root())
}

View File

@ -6,6 +6,7 @@ Une **Edge App** est une application capable de s'exécuter dans un environnemen
### Référence
- [Fichier `manifest.yml`](./apps/manifest.md)
- [API Client](./apps/client-api/README.md)
- [API Serveur](./apps/server-api/README.md)

View File

@ -1,64 +1,14 @@
# API Client
## Méthodes
## Usage
### `Edge.connect(): Promise`
Afin de pouvoir utiliser le SDK "client", vous devez inclure dans la page HTML de votre application la balise `<script>` suivante:
> `TODO`
### `Edge.disconnect(): void`
> `TODO`
### `Edge.send(message: Object): void`
> `TODO`
### `Edge.rpc(method: string, params: Object): Promise`
> `TODO`
#### Exemple
**Côté serveur**
```js
function onInit() {
rpc.register(echo);
}
function echo(ctx, params) {
return params;
}
```html
<script src="/edge/sdk/client.js"></script>
```
**Côté client**
Vous pourrez ensuite accéder aux variables globales suivantes:
```js
Edge.connect().then(() => {
Edge.rpc("echo", { hello: "world!" })
.then(result => console.log(result))
.catch(err => console.error(err));
});
```
### `Edge.upload(blob: Blob, metadata: Object): Promise`
> `TODO`
### `Edge.blobUrl(bucketName: string, blobId: string): string`
> `TODO`
## Événements
### `"message"`
> `TODO`
#### Exemple
```js
Edge.addEventListener("message", evt => console.log(evt.detail));
```
- [`Edge`](./edge.md) - Client principal d'échange avec le serveur
- [`EdgeFrame`](./edge-frame.md)

View File

@ -0,0 +1,30 @@
# `EdgeFrame`
## Méthodes
### `EdgeFrame.addEventListener(name: string, listener: (event) => void)`
> `TODO`
## Événements
### `"title_changed"`
```typescript
interface TitleChangedEvent {
detail: {
title: string
}
}
```
### `"size_changed"`
```typescript
interface SizeChangedEvent {
detail: {
width: number
height: number
}
}
```

View File

@ -0,0 +1,68 @@
# `Edge`
## Méthodes
### `Edge.connect(): Promise`
> `TODO`
### `Edge.disconnect(): void`
> `TODO`
### `Edge.send(message: Object): void`
> `TODO`
### `Edge.rpc(method: string, params: Object): Promise`
> `TODO`
#### Exemple
**Côté serveur**
```js
function onInit() {
rpc.register(echo);
}
function echo(ctx, params) {
return params;
}
```
**Côté client**
```js
Edge.connect().then(() => {
Edge.rpc("echo", { hello: "world!" })
.then(result => console.log(result))
.catch(err => console.error(err));
});
```
### `Edge.upload(blob: Blob, metadata: Object): Promise`
> `TODO`
### `Edge.blobUrl(bucketName: string, blobId: string): string`
> `TODO`
### `Edge.externalUrl(url: string): string`
Retourne une URL "locale" permettant d'accéder à une ressource externe, en fonction de règles propres à l'application. Voir module [`fetch`](../server-api/fetch.md).
## Événements
### `"message"`
> `TODO`
#### Exemple
```js
Edge.addEventListener("message", evt => console.log(evt.detail));
```

36
doc/apps/manifest.md Normal file
View File

@ -0,0 +1,36 @@
# Le fichier `manifest.yml`
Le fichier `manifest.yml` à la racine du bundle de votre application contient des informations décrivant celles ci. Vous trouverez ci dessous un exemple commenté.
```yaml
# REQUIS - L'identifiant de votre application. Il doit être globalement unique.
# Un identifiant du type nom de domaine inversé est en général conseillé (ex: tld.mycompany.myapp)
id: tld.mycompany.myapp
# REQUIS - Le numéro de version de votre application
# Celui ci devrait respecter le format "semver 2" (voir https://semver.org/)
version: 0.0.0
# REQUIS - Le titre de votre application.
title: My App
# OPTIONNEL - Les mots-clés associés à votre applications.
tags: ["chat"]
# OPTIONNEL - La description de votre application.
# Vous pouvez utiliser la syntaxe Markdown pour la mettre en forme.
description: |>
A simple demo application
# OPTIONNEL - Métadonnées associées à l'application
metadata:
# OPTIONNEL - Liste des chemins permettant d'accéder à certains URLs identifiées (page d'administration, icône si existante, etc)
paths:
# Si défini, chemin vers la page d'administration de l'application
admin: /admin
# Si défini, chemin vers l'icône associée à l'application
icon: /my-app-icon.png
# OPTIONNEL - Role minimum requis pour pouvoir accéder à l'application
minimumRole: visitor
```

View File

@ -22,23 +22,7 @@ my-app
Ce fichier est le manifeste de votre application. Il permet au serveur d'identifier celle ci et de récupérer des informations la concernant.
```yaml
---
# L'identifiant de votre application. Il doit être globalement unique.
# Un identifiant du type nom de domaine inversé est en général conseillé (ex: tld.mycompany.myapp)
id: tld.mycompany.myapp
# Le titre de votre application.
title: My App
# Les mots-clés associés à votre applications.
tags: ["chat"]
# La description de votre application.
# Vous pouvez utiliser la syntaxe Markdown pour la mettre en forme.
description: |>
A simple demo application
```
[Voir le fichier `manifest.yml` d'exemple](./manifest.md)
## 4. Créer la page d'accueil

View File

@ -20,11 +20,13 @@ function onInit() {
Listes des modules disponibles côté serveur.
- [`app`](./app.md)
- [`auth`](./auth.md)
- [`blob`](./blob.md)
- [`cast`](./cast.md)
- [`console`](./console.md)
- [`context`](./context.md)
- [`fetch`](./fetch.md)
- [`net`](./net.md)
- [`rpc`](./rpc.md)
- [`store`](./store.md)

View File

@ -0,0 +1,59 @@
# Module `app`
Ce module permet de récupérer des informations sur les applications actives dans l'environnement Edge courant.
## Méthodes
### `app.list(ctx: Context): []Manifest`
Récupère la liste des applications actives.
#### Arguments
- `ctx` **Context** Le contexte d'exécution. Voir la documentation du module [`context`](./context.md)
#### Valeur de retour
Liste des objets `Manifest` décrivant chaque application active.
### `app.get(ctx: Context, appId: string): Manifest`
Récupère les informations de l'application identifiée par `appId`.
#### Arguments
- `ctx` **Context** Le contexte d'exécution. Voir la documentation du module [`context`](./context.md)
- `appId` **string** Identifiant de l'application
#### Valeur de retour
Objet `Manifest` associé à l'application, ou `null` si aucune application n'a été trouvée correspondant à l'identifiant.
### `app.getUrl(ctx: Context, appId: string, from: string = ''): Manifest`
Retourne l'URL permettant d'accéder à l'application identifiée par `appId`.
#### Arguments
- `ctx` **Context** Le contexte d'exécution. Voir la documentation du module [`context`](./context.md)
- `appId` **string** Identifiant de l'application
- `from` **string** Adresse IP qui accédera à l'application (permet de générer la bonne URL vis à vis du réseau d'origine)
#### Valeur de retour
URL associée à l'application, ou `null` si aucune application n'a été trouvée correspondant à l'identifiant.
## Objets
### `Manifest`
```typescript
interface Manifest {
id: string // Identifiant de l'application
version: string // Version de l'application
title: string // Titre associé à l'application
description: string // Description associée à l'application
tags: string[] // Mots clés associés à l'application
metadata: { [key: string]: any } // Métadonnées associées à l'application. Voir ../manifest.md
}
```

View File

@ -38,11 +38,15 @@ function onBlobDownload(ctx, bucketName, blobId) {
> `TODO`
### `blob.writeBlob(ctx: Context, bucketName: string, blobId: string)`
### `blob.getBlobInfo(ctx: Context, bucketName: string, blobId: string): BlobInfo`
> `TODO`
### `blob.readBlob(ctx: Context, bucketName: string, blobId: string)`
### `blob.writeBlob(ctx: Context, bucketName: string, blobId: string, data: any)`
> `TODO`
### `blob.readBlob(ctx: Context, bucketName: string, blobId: string): ArrayBuffer`
> `TODO`
@ -58,7 +62,7 @@ function onBlobDownload(ctx, bucketName, blobId) {
> `TODO`
### `blob.getBlobInfo(ctx: Context, bucketName: string, blobId: string): BlobInfo`
### `blob.getBucketSize(ctx: Context, bucketName: string): number`
> `TODO`
@ -70,4 +74,16 @@ Voir la documentation de l'objet [`Context`](./context.md#Context).
### `BlobInfo`
### `Metadata`
```typescript
interface BlobInfo {
id: string // Identifiant du blob
bucket: string // Nom du bucket contenant le blob
size: number // Taille du blob
modTime: number // Timestamp Unix de dernière modification du blob
contentType: string // Type MIME du contenu du blob
}
```
### `Metadata`
L'objet `Metadata` est un objet clé/valeur arbitraire transmis avec la requête de téléversement. Voir la méthode [`Edge.upload(blob, metadata)`](../client-api/README.md#edge-upload-blob-blob-metadata-object-promise) du SDK client.

View File

@ -0,0 +1,33 @@
# Module `fetch`
Ce module permet l'accès à des ressources distantes (sur Internet) depuis votre application.
## Fonctions de rappel
Pour permettre aux utilisateurs d'accéder à des ressources distantes, vous devez déclarer la fonction `onClientFetch(ctx: Context, url: string, remoteAddr: string)` dans le fichier `server/main.js` de votre application.
### `onClientFetch(ctx: Context, url: string, remoteAddr: string)`
#### Usage
**Côté client**
```js
// Création d'une URL "locale" permettant d'accéder à la ressource distante
var url = Edge.externalUrl("http://example.com")
// Vous pouvez utiliser l'URL comme attribut `src` d'une balise <img> par exemple
// ou effectuer une requête fetch() avec celle ci.
fetch(url).then(res => res.text()).then(content => console.log(content));
```
**Côté serveur**
```js
function onClientFetch(ctx, url, remoteAddr) {
// Autoriser la récupération de l'URL demandée ou non
// Dans cet exemple, seule l'URL externe 'http://example.com' est autorisée
// Les autres URLs recevront une erreur HTTP 403 - Forbidden
var authorized = url === "http://example.com"
return { allow: authorized };
}
```

3
go.mod
View File

@ -8,6 +8,7 @@ require (
)
require (
github.com/brutella/dnssd v1.2.6 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/goccy/go-json v0.9.11 // indirect
github.com/gogo/protobuf v0.0.0-20161014173244-50d1bd39ce4e // indirect
@ -19,7 +20,7 @@ require (
github.com/lestrrat-go/httprc v1.0.4 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.0 // indirect
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8 // indirect
github.com/miekg/dns v1.1.50 // indirect
)
require (

13
go.sum
View File

@ -54,6 +54,8 @@ github.com/alecthomas/kong-hcl v0.1.8-0.20190615233001-b21fea9723c8/go.mod h1:MR
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692 h1:JW4WZlqyaNWUUahfr7MigeDW6jmtam5cTzzo1lwsFhE=
github.com/barnybug/go-cast v0.0.0-20201201064555-a87ccbc26692/go.mod h1:Au0ipPuCBA7zsOC61SnyrYetm8VT3vo1UJtwHeYke44=
github.com/brutella/dnssd v1.2.6 h1:/0P13JkHLRzeLQkWRPEn4hJCr4T3NfknIFw3aNPIC34=
github.com/brutella/dnssd v1.2.6/go.mod h1:JoW2sJUrmVIef25G6lrLj7HS6Xdwh6q8WUIvMkkBYXs=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@ -232,6 +234,8 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8 h1:ALvJ9V8nNf04PFHMR2sot56N/pjrx5LzZGvUlnhdiCE=
github.com/miekg/dns v0.0.0-20161006100029-fc4e1e2843d8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA=
github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
@ -283,6 +287,7 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
gitlab.com/wpetit/goweb v0.0.0-20230206085656-dec695f0e2e9 h1:6JlkcdjYVQglPWYuemK2MoZAtRE4vFx85zLXflGIyI8=
gitlab.com/wpetit/goweb v0.0.0-20230206085656-dec695f0e2e9/go.mod h1:3sus4zjoUv1GB7eDLL60QaPkUnXJCWBpjvbe0jWifeY=
@ -340,6 +345,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
@ -376,6 +382,8 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@ -402,6 +410,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -439,10 +448,13 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -521,6 +533,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=

View File

@ -4,4 +4,9 @@ title: SDK Test
version: 0.0.0
description: |
Suite de tests pour le SDK client
tags: ["test"]
tags: ["test"]
metadata:
paths:
icon: /icon.png
minimumRole: visitor

Binary file not shown.

After

Width:  |  Height:  |  Size: 77 KiB

View File

@ -4,6 +4,7 @@
<meta charset="utf-8" />
<title>Client SDK Test suite</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" type="image/png" href="/icon.png">
<link rel="stylesheet" href="/vendor/mocha.css" />
<style>
body {
@ -25,6 +26,8 @@
<script src="/test/net-module.js"></script>
<script src="/test/rpc-module.js"></script>
<script src="/test/file-module.js"></script>
<script src="/test/app-module.js"></script>
<script src="/test/fetch-module.js"></script>
<script class="mocha-exec">
mocha.run();
</script>

View File

@ -0,0 +1,45 @@
describe('App Module', function() {
before(() => {
return Edge.connect();
});
after(() => {
Edge.disconnect();
});
it('should list apps', function() {
return Edge.rpc("listApps")
.then(apps => {
console.log("listApps result:", apps);
chai.assert.isNotNull(apps);
chai.assert.isAtLeast(apps.length, 1);
})
});
it('should retrieve requested app', function() {
return Edge.rpc("getApp", { appId: "edge.sdk.client.test" })
.then(app => {
console.log("getApp result:", app);
chai.assert.isNotNull(app);
chai.assert.equal(app.id, "edge.sdk.client.test");
})
});
it('should retrieve requested app url without from address', function() {
return Edge.rpc("getAppUrl", { appId: "edge.sdk.client.test" })
.then(url => {
console.log("getAppUrl result:", url);
chai.assert.isNotEmpty(url);
})
});
it('should retrieve requested app url with from address', function() {
return Edge.rpc("getAppUrl", { appId: "edge.sdk.client.test", from: "127.0.0.2" })
.then(url => {
console.log("getAppUrl result:", url);
chai.assert.isNotEmpty(url);
})
});
});

View File

@ -1,4 +1,5 @@
Edge.debug = true;
EdgeFrame.debug = true;
describe('Edge', function() {

View File

@ -0,0 +1,33 @@
describe('Fetch Module', function () {
before(() => {
return Edge.connect();
});
after(() => {
Edge.disconnect();
});
it('should fetch an authorized external url', function () {
var externalUrl = Edge.externalUrl("http://example.com");
return fetch(externalUrl)
.then(res => {
chai.assert.equal(res.status, 200)
return res.text()
})
.then(content => {
chai.assert.include(content, '<h1>Example Domain</h1>')
})
});
it('should not fetch an unauthorized external url', function () {
var externalUrl = Edge.externalUrl("https://google.com");
return fetch(externalUrl)
.then(res => {
chai.assert.equal(res.status, 403)
})
});
});

View File

@ -38,7 +38,7 @@ describe('Remote Procedure Call', function () {
it('should call the add() method repetitively and keep count of the sent values', function () {
this.timeout(10000);
this.timeout(30000);
const values = [];
for (let i = 0; i <= 1000; i++) {

View File

@ -11,6 +11,10 @@ function onInit() {
rpc.register("reset", reset);
rpc.register("total", total);
rpc.register("getUserInfo", getUserInfo);
rpc.register("listApps");
rpc.register("getApp");
rpc.register("getAppUrl");
}
// Called for each client message
@ -79,4 +83,24 @@ function getUserInfo(ctx, params) {
role: role,
preferredUsername: preferredUsername,
};
}
function listApps(ctx) {
return app.list(ctx);
}
function getApp(ctx, params) {
var appId = params.appId;
return app.get(ctx, appId);
}
function getAppUrl(ctx, params) {
var appId = params.appId;
var from = params.from;
return app.getUrl(ctx, appId, from ? from : '');
}
function onClientFetch(ctx, url, remoteAddr) {
return { allow: url === 'http://example.com' };
}

28
misc/jenkins/Dockerfile Normal file
View File

@ -0,0 +1,28 @@
FROM reg.cadoles.com/proxy_cache/library/ubuntu:22.04
ARG HTTP_PROXY=
ARG HTTPS_PROXY=
ARG http_proxy=
ARG https_proxy=
ARG GO_VERSION=1.19.2
# Install dev environment dependencies
RUN export DEBIAN_FRONTEND=noninteractive &&\
apt-get update -y &&\
apt-get install -y --no-install-recommends curl ca-certificates build-essential wget unzip tar git jq
# Install Go
RUN mkdir -p /tmp \
&& wget -O /tmp/go${GO_VERSION}.linux-amd64.tar.gz https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz \
&& rm -rf /usr/local/go \
&& mkdir -p /usr/local \
&& tar -C /usr/local -xzf /tmp/go${GO_VERSION}.linux-amd64.tar.gz
ENV PATH="${PATH}:/usr/local/go/bin"
# Add LetsEncrypt certificates
RUN curl -k https://forge.cadoles.com/Cadoles/Jenkins/raw/branch/master/resources/com/cadoles/common/add-letsencrypt-ca.sh | bash
# Install NodeJS
RUN curl -fsSL https://deb.nodesource.com/setup_18.x | bash - \
&& apt-get install -y nodejs

View File

@ -7,7 +7,7 @@ modd.conf
{
prep: make build-sdk
prep: cd misc/client-sdk-testsuite && make dist
prep: make GOTEST_ARGS="-short" test
prep: make build
prep: make GOTEST_ARGS="-short" test
daemon: bin/cli app run -p misc/client-sdk-testsuite/dist
}

View File

@ -1,39 +0,0 @@
package app
import (
"forge.cadoles.com/arcad/edge/pkg/bundle"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
type ID string
type Manifest struct {
ID ID `yaml:"id"`
Version string `yaml:"version"`
Title string `yaml:"title"`
Description string `yaml:"description"`
Tags []string `yaml:"tags"`
}
func LoadManifest(b bundle.Bundle) (*Manifest, error) {
reader, _, err := b.File("manifest.yml")
if err != nil {
return nil, errors.Wrap(err, "could not read manifest.yml")
}
defer func() {
if err := reader.Close(); err != nil {
panic(errors.WithStack(err))
}
}()
manifest := &Manifest{}
decoder := yaml.NewDecoder(reader)
if err := decoder.Decode(manifest); err != nil {
return nil, errors.Wrap(err, "could not decode manifest.yml")
}
return manifest, nil
}

85
pkg/app/manifest.go Normal file
View File

@ -0,0 +1,85 @@
package app
import (
"strings"
"forge.cadoles.com/arcad/edge/pkg/bundle"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
"gopkg.in/yaml.v2"
)
type ID string
type Manifest struct {
ID ID `yaml:"id" json:"id"`
Version string `yaml:"version" json:"version"`
Title string `yaml:"title" json:"title"`
Description string `yaml:"description" json:"description"`
Tags []string `yaml:"tags" json:"tags"`
Metadata MapStr `yaml:"metadata" json:"metadata"`
}
type MetadataValidator func(map[string]any) (bool, error)
func (m *Manifest) Validate(validators ...MetadataValidator) (bool, error) {
if m.ID == "" {
return false, errors.New("'id' property should not be empty")
}
if m.Version == "" {
return false, errors.New("'version' property should not be empty")
}
version := m.Version
if !strings.HasPrefix(version, "v") {
version = "v" + version
}
if !semver.IsValid(version) {
return false, errors.Errorf("version '%s' does not respect semver format", m.Version)
}
if m.Title == "" {
return false, errors.New("'title' property should not be empty")
}
if m.Tags != nil {
for _, t := range m.Tags {
if strings.ContainsAny(t, " \t\n\r") {
return false, errors.Errorf("tag '%s' should not contain any space or new line", t)
}
}
}
for _, v := range validators {
valid, err := v(m.Metadata)
if !valid || err != nil {
return valid, errors.WithStack(err)
}
}
return true, nil
}
func LoadManifest(b bundle.Bundle) (*Manifest, error) {
reader, _, err := b.File("manifest.yml")
if err != nil {
return nil, errors.Wrap(err, "could not read manifest.yml")
}
defer func() {
if err := reader.Close(); err != nil {
panic(errors.WithStack(err))
}
}()
manifest := &Manifest{}
decoder := yaml.NewDecoder(reader)
if err := decoder.Decode(manifest); err != nil {
return nil, errors.Wrap(err, "could not decode manifest.yml")
}
return manifest, nil
}

61
pkg/app/map_str.go Normal file
View File

@ -0,0 +1,61 @@
package app
import (
"fmt"
"github.com/pkg/errors"
)
type MapStr map[string]interface{}
func MapStrUnion(dict1 MapStr, dict2 MapStr) MapStr {
dict := MapStr{}
for k, v := range dict1 {
dict[k] = v
}
for k, v := range dict2 {
dict[k] = v
}
return dict
}
func (ms *MapStr) UnmarshalYAML(unmarshal func(interface{}) error) error {
var result map[interface{}]interface{}
err := unmarshal(&result)
if err != nil {
return errors.WithStack(err)
}
*ms = cleanUpInterfaceMap(result)
return nil
}
func cleanUpInterfaceArray(in []interface{}) []interface{} {
result := make([]interface{}, len(in))
for i, v := range in {
result[i] = cleanUpMapValue(v)
}
return result
}
func cleanUpInterfaceMap(in map[interface{}]interface{}) MapStr {
result := make(MapStr)
for k, v := range in {
result[fmt.Sprintf("%v", k)] = cleanUpMapValue(v)
}
return result
}
func cleanUpMapValue(v interface{}) interface{} {
switch v := v.(type) {
case []interface{}:
return cleanUpInterfaceArray(v)
case map[interface{}]interface{}:
return cleanUpInterfaceMap(v)
case string:
return v
default:
return fmt.Sprintf("%v", v)
}
}

View File

@ -0,0 +1,28 @@
package metadata
import (
"forge.cadoles.com/arcad/edge/pkg/app"
"github.com/pkg/errors"
)
func WithMinimumRoleValidator(roles ...string) app.MetadataValidator {
return func(metadata map[string]any) (bool, error) {
rawMinimumRole, exists := metadata["minimumRole"]
if !exists {
return true, nil
}
minimumRole, ok := rawMinimumRole.(string)
if !ok {
return false, errors.Errorf("metadata['minimumRole']: unexpected value type '%T'", rawMinimumRole)
}
for _, r := range roles {
if minimumRole == r {
return true, nil
}
}
return false, errors.Errorf("metadata['minimumRole']: unexpected role '%s'", minimumRole)
}
}

View File

@ -0,0 +1,51 @@
package metadata
import (
"strings"
"forge.cadoles.com/arcad/edge/pkg/app"
"github.com/pkg/errors"
)
type NamedPath string
const (
NamedPathAdmin NamedPath = "admin"
NamedPathIcon NamedPath = "icon"
)
func WithNamedPathsValidator(names ...NamedPath) app.MetadataValidator {
set := map[NamedPath]struct{}{}
for _, n := range names {
set[n] = struct{}{}
}
return func(metadata map[string]any) (bool, error) {
rawPaths, exists := metadata["paths"]
if !exists {
return true, nil
}
paths, ok := rawPaths.(app.MapStr)
if !ok {
return false, errors.Errorf("metadata['paths']: unexpected named path value type '%T'", rawPaths)
}
for n, p := range paths {
if _, exists := set[NamedPath(n)]; !exists {
return false, errors.Errorf("metadata['paths']: unexpected named path '%s'", n)
}
path, ok := p.(string)
if !ok {
return false, errors.Errorf("metadata['paths']['%s']: unexpected named path value type '%T'", n, path)
}
if !strings.HasPrefix(path, "/") {
return false, errors.Errorf("metadata['paths']['%s']: named path value should start with a '/'", n)
}
}
return true, nil
}
}

View File

@ -0,0 +1,7 @@
id: foo.arcad.app
version: v0.0.0
title: Foo
description: A test app
tags: ["test"]
metadata:
minimumRole: foo

View File

@ -0,0 +1,10 @@
id: foo.arcad.app
version: v0.0.0
title: Foo
description: A test app
tags: ["test"]
metadata:
paths:
invalid: /admin
icon: /my-app-icon.png
minimumRole: visitor

View File

@ -0,0 +1,10 @@
id: foo.arcad.app
version: v0.0.0
title: Foo
description: A test app
tags: ["test"]
metadata:
paths:
admin: /admin
icon: /my-app-icon.png
minimumRole: visitor

View File

@ -0,0 +1,74 @@
package metadata
import (
"io/ioutil"
"path/filepath"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"
)
type validatorTestCase struct {
File string
ExpectValid bool
ExpectError bool
}
var validatorTestCases = []validatorTestCase{
{
File: "valid.yml",
ExpectValid: true,
},
{
File: "invalid-paths.yml",
ExpectValid: false,
ExpectError: true,
},
{
File: "invalid-minimum-role.yml",
ExpectValid: false,
ExpectError: true,
},
}
var validators = []app.MetadataValidator{
WithMinimumRoleValidator("visitor", "user", "superuser", "admin", "superadmin"),
WithNamedPathsValidator(NamedPathAdmin, NamedPathIcon),
}
func TestManifestValidate(t *testing.T) {
for _, tc := range validatorTestCases {
func(tc *validatorTestCase) {
t.Run(tc.File, func(t *testing.T) {
data, err := ioutil.ReadFile(filepath.Join("testdata/manifests", tc.File))
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
var manifest app.Manifest
if err := yaml.Unmarshal(data, &manifest); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
valid, err := manifest.Validate(validators...)
t.Logf("[RESULT] valid:%v, err:%v", valid, err)
if e, g := tc.ExpectValid, valid; e != g {
t.Errorf("valid: expected '%v', got '%v'", e, g)
}
if tc.ExpectError && err == nil {
t.Error("err should not be nil")
}
if !tc.ExpectError && err != nil {
t.Errorf("err: expected nil, got '%+v'", err)
}
})
}(&tc)
}
}

View File

@ -1,12 +1,9 @@
package memory
import (
"context"
"sync"
"time"
"forge.cadoles.com/arcad/edge/pkg/bus"
"gitlab.com/wpetit/goweb/logger"
)
type eventDispatcherSet struct {
@ -89,8 +86,6 @@ func (d *eventDispatcher) IsOut(out <-chan bus.Message) bool {
}
func (d *eventDispatcher) Run() {
ctx := context.Background()
for {
msg, ok := <-d.in
if !ok {
@ -99,12 +94,7 @@ func (d *eventDispatcher) Run() {
return
}
timeout := time.After(2 * time.Second)
select {
case d.out <- msg:
case <-timeout:
logger.Error(ctx, "message out chan timed out", logger.F("message", msg))
}
d.out <- msg
}
}

View File

@ -11,6 +11,7 @@ import (
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/module/blob"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/go-chi/chi/v5"
"github.com/pkg/errors"
@ -68,7 +69,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
ContextKeyOriginRequest: r,
})
requestMsg := module.NewMessageUploadRequest(ctx, fileHeader, metadata)
requestMsg := blob.NewMessageUploadRequest(ctx, fileHeader, metadata)
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
@ -80,7 +81,7 @@ func (h *Handler) handleAppUpload(w http.ResponseWriter, r *http.Request) {
logger.Debug(ctx, "upload reply", logger.F("reply", reply))
responseMsg, ok := reply.(*module.MessageUploadResponse)
responseMsg, ok := reply.(*blob.MessageUploadResponse)
if !ok {
logger.Error(
ctx, "unexpected upload response message",
@ -120,7 +121,7 @@ func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
ContextKeyOriginRequest: r,
})
requestMsg := module.NewMessageDownloadRequest(ctx, bucket, storage.BlobID(blobID))
requestMsg := blob.NewMessageDownloadRequest(ctx, bucket, storage.BlobID(blobID))
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
@ -130,7 +131,7 @@ func (h *Handler) handleAppDownload(w http.ResponseWriter, r *http.Request) {
return
}
replyMsg, ok := reply.(*module.MessageDownloadResponse)
replyMsg, ok := reply.(*blob.MessageDownloadResponse)
if !ok {
logger.Error(
ctx, "unexpected download response message",

112
pkg/http/fetch.go Normal file
View File

@ -0,0 +1,112 @@
package http
import (
"io"
"net/http"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
h.mutex.RLock()
defer h.mutex.RUnlock()
ctx := r.Context()
ctx = module.WithContext(ctx, map[module.ContextKey]any{
ContextKeyOriginRequest: r,
})
rawURL := r.URL.Query().Get("url")
url, err := url.Parse(rawURL)
if err != nil {
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
return
}
requestMsg := fetch.NewMessageFetchRequest(ctx, r.RemoteAddr, url)
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
logger.Error(ctx, "could not retrieve fetch request reply", logger.E(errors.WithStack(err)))
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
logger.Debug(ctx, "fetch reply", logger.F("reply", reply))
responseMsg, ok := reply.(*fetch.MessageFetchResponse)
if !ok {
logger.Error(
ctx, "unexpected fetch response message",
logger.F("message", reply),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
if !responseMsg.Allow {
jsonError(w, http.StatusForbidden, errorCodeForbidden)
return
}
proxyReq, err := http.NewRequest(http.MethodGet, url.String(), nil)
if err != nil {
logger.Error(
ctx, "could not create proxy request",
logger.E(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
proxyReq.Header.Add("X-Forwarded-From", r.RemoteAddr)
res, err := h.httpClient.Do(proxyReq)
if err != nil {
logger.Error(
ctx, "could not execute proxy request",
logger.E(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
defer func() {
if err := res.Body.Close(); err != nil {
logger.Error(
ctx, "could not close response body",
logger.E(errors.WithStack(err)),
)
}
}()
for header, values := range res.Header {
for _, value := range values {
w.Header().Add(header, value)
}
}
w.WriteHeader(res.StatusCode)
if _, err := io.Copy(w, res.Body); err != nil {
panic(errors.WithStack(err))
}
}

View File

@ -31,6 +31,8 @@ type Handler struct {
server *app.Server
serverModuleFactories []app.ServerModuleFactory
httpClient *http.Client
mutex sync.RWMutex
}
@ -91,6 +93,7 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
sockjsOpts: opts.SockJS,
router: router,
serverModuleFactories: opts.ServerModuleFactories,
httpClient: opts.HTTPClient,
bus: opts.Bus,
}
@ -103,6 +106,8 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
r.Route("/api/v1", func(r chi.Router) {
r.Post("/upload", handler.handleAppUpload)
r.Get("/download/{bucket}/{blobID}", handler.handleAppDownload)
r.Get("/fetch", handler.handleAppFetch)
})
r.HandleFunc("/sock/*", handler.handleSockJS)

View File

@ -1,6 +1,7 @@
package http
import (
"net/http"
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
@ -14,6 +15,7 @@ type HandlerOptions struct {
SockJS sockjs.Options
ServerModuleFactories []app.ServerModuleFactory
UploadMaxFileSize int64
HTTPClient *http.Client
}
func defaultHandlerOptions() *HandlerOptions {
@ -27,6 +29,9 @@ func defaultHandlerOptions() *HandlerOptions {
SockJS: sockjsOptions,
ServerModuleFactories: make([]app.ServerModuleFactory, 0),
UploadMaxFileSize: 10 << (10 * 2), // 10Mb
HTTPClient: &http.Client{
Timeout: time.Second * 30,
},
}
}
@ -55,3 +60,9 @@ func WithUploadMaxFileSize(size int64) HandlerOptionFunc {
opts.UploadMaxFileSize = size
}
}
func WithHTTPClient(client *http.Client) HandlerOptionFunc {
return func(opts *HandlerOptions) {
opts.HTTPClient = client
}
}

5
pkg/module/app/error.go Normal file
View File

@ -0,0 +1,5 @@
package app
import "errors"
var ErrNotFound = errors.New("not found")

View File

@ -0,0 +1,58 @@
package memory
import (
"context"
"fmt"
"io/ioutil"
"testing"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module"
appModule "forge.cadoles.com/arcad/edge/pkg/module/app"
"github.com/pkg/errors"
)
func TestAppModuleWithMemoryRepository(t *testing.T) {
t.Parallel()
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
appModule.ModuleFactory(NewRepository(
func(ctx context.Context, id app.ID, from string) (string, error) {
return fmt.Sprintf("http//%s.example.com?from=%s", id, from), nil
},
&app.Manifest{
ID: "dummy1.arcad.app",
Version: "0.0.0",
Title: "Dummy 1",
Description: "Dummy App 1",
Tags: []string{"dummy", "first"},
},
&app.Manifest{
ID: "dummy2.arcad.app",
Version: "0.0.0",
Title: "Dummy 2",
Description: "Dummy App 2",
Tags: []string{"dummy", "second"},
},
)),
)
file := "testdata/app.js"
data, err := ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if err := server.Load(file, string(data)); err != nil {
t.Fatal(err)
}
defer server.Stop()
if err := server.Start(); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
}

View File

@ -0,0 +1,50 @@
package memory
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
module "forge.cadoles.com/arcad/edge/pkg/module/app"
"github.com/pkg/errors"
)
type GetURLFunc func(context.Context, app.ID, string) (string, error)
type Repository struct {
getURL GetURLFunc
apps []*app.Manifest
}
// GetURL implements app.Repository
func (r *Repository) GetURL(ctx context.Context, id app.ID, from string) (string, error) {
url, err := r.getURL(ctx, id, from)
if err != nil {
return "", errors.WithStack(err)
}
return url, nil
}
// Get implements app.Repository
func (r *Repository) Get(ctx context.Context, id app.ID) (*app.Manifest, error) {
for _, app := range r.apps {
if app.ID != id {
continue
}
return app, nil
}
return nil, module.ErrNotFound
}
// List implements app.Repository
func (r *Repository) List(ctx context.Context) ([]*app.Manifest, error) {
return r.apps, nil
}
func NewRepository(getURL GetURLFunc, manifests ...*app.Manifest) *Repository {
return &Repository{getURL, manifests}
}
var _ module.Repository = &Repository{}

17
pkg/module/app/memory/testdata/app.js vendored Normal file
View File

@ -0,0 +1,17 @@
var ctx = context.new();
var manifests = app.list(ctx);
if (manifests.length !== 2) {
throw new Error("apps.length: expected '2', got '"+manifests.length+"'");
}
var manifest = app.get(ctx, 'dummy2.arcad.app');
if (!manifest) {
throw new Error("manifest should not be null");
}
if (manifest.id !== "dummy2.arcad.app") {
throw new Error("manifest.id: expected 'dummy2.arcad.app', got '"+manifest.id+"'");
}

124
pkg/module/app/module.go Normal file
View File

@ -0,0 +1,124 @@
package app
import (
"fmt"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/module/util"
"github.com/dop251/goja"
"github.com/pkg/errors"
)
type Module struct {
repository Repository
}
type gojaManifest struct {
ID string `goja:"id" json:"id"`
Version string `goja:"version" json:"version"`
Title string `goja:"title" json:"title"`
Description string `goja:"description" json:"description"`
Tags []string `goja:"tags" json:"tags"`
Metadata map[string]any `goja:"metadata" json:"metadata"`
}
func toGojaManifest(manifest *app.Manifest) *gojaManifest {
return &gojaManifest{
ID: string(manifest.ID),
Version: manifest.Version,
Title: manifest.Title,
Description: manifest.Description,
Tags: manifest.Tags,
Metadata: manifest.Metadata,
}
}
func toGojaManifests(manifests []*app.Manifest) []*gojaManifest {
gojaManifests := make([]*gojaManifest, len(manifests))
for i, m := range manifests {
gojaManifests[i] = toGojaManifest(m)
}
return gojaManifests
}
func (m *Module) Name() string {
return "app"
}
func (m *Module) Export(export *goja.Object) {
if err := export.Set("list", m.list); err != nil {
panic(errors.Wrap(err, "could not set 'list' function"))
}
if err := export.Set("get", m.get); err != nil {
panic(errors.Wrap(err, "could not set 'get' function"))
}
if err := export.Set("getUrl", m.getURL); err != nil {
panic(errors.Wrap(err, "could not set 'list' function"))
}
}
func (m *Module) list(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
manifests, err := m.repository.List(ctx)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(toGojaManifests(manifests))
}
func (m *Module) get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
appID := assertAppID(call.Argument(1), rt)
manifest, err := m.repository.Get(ctx, appID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(toGojaManifest(manifest))
}
func (m *Module) getURL(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
appID := assertAppID(call.Argument(1), rt)
var from string
if len(call.Arguments) > 2 {
from = util.AssertString(call.Argument(2), rt)
}
url, err := m.repository.GetURL(ctx, appID, from)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(url)
}
func ModuleFactory(repository Repository) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
return &Module{
repository: repository,
}
}
}
func assertAppID(value goja.Value, rt *goja.Runtime) app.ID {
appID, ok := value.Export().(app.ID)
if !ok {
rawAppID, ok := value.Export().(string)
if !ok {
panic(rt.NewTypeError(fmt.Sprintf("app id must be an appid or a string, got '%T'", value.Export())))
}
appID = app.ID(rawAppID)
}
return appID
}

View File

@ -0,0 +1,13 @@
package app
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
)
type Repository interface {
List(context.Context) ([]*app.Manifest, error)
Get(context.Context, app.ID) (*app.Manifest, error)
GetURL(context.Context, app.ID, string) (string, error)
}

View File

@ -30,10 +30,12 @@ func init() {
}
type LocalHandler struct {
router chi.Router
algo jwa.KeyAlgorithm
key jwk.Key
accounts map[string]LocalAccount
router chi.Router
algo jwa.KeyAlgorithm
key jwk.Key
getCookieDomain GetCookieDomainFunc
cookieDuration time.Duration
accounts map[string]LocalAccount
}
func (h *LocalHandler) initRouter(prefix string) {
@ -116,10 +118,20 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
return
}
cookieDomain, err := h.getCookieDomain(r)
if err != nil {
logger.Error(ctx, "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
cookie := http.Cookie{
Name: auth.CookieName,
Value: string(token),
Domain: cookieDomain,
HttpOnly: false,
Expires: time.Now().Add(h.cookieDuration),
Path: "/",
}
@ -129,11 +141,20 @@ func (h *LocalHandler) handleForm(w http.ResponseWriter, r *http.Request) {
}
func (h *LocalHandler) handleLogout(w http.ResponseWriter, r *http.Request) {
cookieDomain, err := h.getCookieDomain(r)
if err != nil {
logger.Error(r.Context(), "could not retrieve cookie domain", logger.E(errors.WithStack(err)))
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
return
}
http.SetCookie(w, &http.Cookie{
Name: auth.CookieName,
Value: "",
HttpOnly: false,
Expires: time.Unix(0, 0),
Domain: cookieDomain,
Path: "/",
})
@ -165,9 +186,11 @@ func NewLocalHandler(algo jwa.KeyAlgorithm, key jwk.Key, funcs ...LocalHandlerOp
}
handler := &LocalHandler{
algo: algo,
key: key,
accounts: toAccountsMap(opts.Accounts),
algo: algo,
key: key,
accounts: toAccountsMap(opts.Accounts),
getCookieDomain: opts.GetCookieDomain,
cookieDuration: opts.CookieDuration,
}
handler.initRouter(opts.RoutePrefix)

View File

@ -1,16 +1,31 @@
package http
import (
"net/http"
"time"
)
type GetCookieDomainFunc func(r *http.Request) (string, error)
func defaultGetCookieDomain(r *http.Request) (string, error) {
return "", nil
}
type LocalHandlerOptions struct {
RoutePrefix string
Accounts []LocalAccount
RoutePrefix string
Accounts []LocalAccount
GetCookieDomain GetCookieDomainFunc
CookieDuration time.Duration
}
type LocalHandlerOptionFunc func(*LocalHandlerOptions)
func defaultLocalHandlerOptions() *LocalHandlerOptions {
return &LocalHandlerOptions{
RoutePrefix: "",
Accounts: make([]LocalAccount, 0),
RoutePrefix: "",
Accounts: make([]LocalAccount, 0),
GetCookieDomain: defaultGetCookieDomain,
CookieDuration: 24 * time.Hour,
}
}
@ -25,3 +40,10 @@ func WithRoutePrefix(prefix string) LocalHandlerOptionFunc {
opts.RoutePrefix = prefix
}
}
func WithCookieOptions(getCookieDomain GetCookieDomainFunc, duration time.Duration) LocalHandlerOptionFunc {
return func(opts *LocalHandlerOptions) {
opts.GetCookieDomain = getCookieDomain
opts.CookieDuration = duration
}
}

View File

@ -61,6 +61,10 @@ func FindToken(r *http.Request, getKeySet GetKeySetFunc) (jwt.Token, error) {
return nil, errors.WithStack(err)
}
if keySet == nil {
return nil, errors.New("no keyset")
}
token, err := jwt.Parse([]byte(rawToken),
jwt.WithKeySet(keySet, jws.WithRequireKid(false)),
jwt.WithValidate(true),

View File

@ -1,282 +0,0 @@
package module
import (
"context"
"io"
"mime/multipart"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/dop251/goja"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
const (
DefaultBlobBucket string = "default"
)
type BlobModule struct {
server *app.Server
bus bus.Bus
store storage.BlobStore
}
func (m *BlobModule) Name() string {
return "blob"
}
func (m *BlobModule) Export(export *goja.Object) {
}
func (m *BlobModule) handleMessages() {
ctx := context.Background()
go func() {
err := m.bus.Reply(ctx, MessageNamespaceUploadRequest, func(msg bus.Message) (bus.Message, error) {
uploadRequest, ok := msg.(*MessageUploadRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message upload request, got '%T'", msg)
}
res, err := m.handleUploadRequest(uploadRequest)
if err != nil {
logger.Error(ctx, "could not handle upload request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
logger.Debug(ctx, "upload request response", logger.F("response", res))
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}()
err := m.bus.Reply(ctx, MessageNamespaceDownloadRequest, func(msg bus.Message) (bus.Message, error) {
downloadRequest, ok := msg.(*MessageDownloadRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message download request, got '%T'", msg)
}
res, err := m.handleDownloadRequest(downloadRequest)
if err != nil {
logger.Error(ctx, "could not handle download request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}
func (m *BlobModule) handleUploadRequest(req *MessageUploadRequest) (*MessageUploadResponse, error) {
blobID := storage.NewBlobID()
res := NewMessageUploadResponse(req.RequestID)
ctx := logger.With(req.Context, logger.F("blobID", blobID))
blobInfo := map[string]interface{}{
"size": req.FileHeader.Size,
"filename": req.FileHeader.Filename,
"contentType": req.FileHeader.Header.Get("Content-Type"),
}
rawResult, err := m.server.ExecFuncByName(ctx, "onBlobUpload", ctx, blobID, blobInfo, req.Metadata)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onBlobUpload result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
if res.Allow {
bucket := DefaultBlobBucket
rawBucket, exists := result["bucket"]
if exists {
bucket, ok = rawBucket.(string)
if !ok {
return nil, errors.Errorf("invalid 'bucket' result property: got type '%T', expected type '%T'", bucket, "")
}
}
if err := m.saveBlob(ctx, bucket, blobID, *req.FileHeader); err != nil {
return nil, errors.WithStack(err)
}
res.Bucket = bucket
res.BlobID = blobID
}
return res, nil
}
func (m *BlobModule) saveBlob(ctx context.Context, bucketName string, blobID storage.BlobID, fileHeader multipart.FileHeader) error {
file, err := fileHeader.Open()
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
}
}()
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
}
}()
writer, err := bucket.NewWriter(ctx, blobID)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
}
}()
defer func() {
if err := writer.Close(); err != nil {
logger.Error(ctx, "could not close writer", logger.E(errors.WithStack(err)))
}
}()
if _, err := io.Copy(writer, file); err != nil {
return errors.WithStack(err)
}
return nil
}
func (m *BlobModule) handleDownloadRequest(req *MessageDownloadRequest) (*MessageDownloadResponse, error) {
res := NewMessageDownloadResponse(req.RequestID)
rawResult, err := m.server.ExecFuncByName(req.Context, "onBlobDownload", req.Context, req.Bucket, req.BlobID)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onBlobDownload result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
reader, info, err := m.openBlob(req.Context, req.Bucket, req.BlobID)
if err != nil && !errors.Is(err, storage.ErrBlobNotFound) {
return nil, errors.WithStack(err)
}
if reader != nil {
res.Blob = reader
}
if info != nil {
res.BlobInfo = info
}
return res, nil
}
func (m *BlobModule) openBlob(ctx context.Context, bucketName string, blobID storage.BlobID) (io.ReadSeekCloser, storage.BlobInfo, error) {
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
return nil, nil, errors.WithStack(err)
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)), logger.F("bucket", bucket))
}
}()
info, err := bucket.Get(ctx, blobID)
if err != nil {
return nil, nil, errors.WithStack(err)
}
reader, err := bucket.NewReader(ctx, blobID)
if err != nil {
return nil, nil, errors.WithStack(err)
}
return reader, info, nil
}
func BlobModuleFactory(bus bus.Bus, store storage.BlobStore) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
mod := &BlobModule{
store: store,
bus: bus,
server: server,
}
go mod.handleMessages()
return mod
}
}

View File

@ -0,0 +1,21 @@
package blob
import "forge.cadoles.com/arcad/edge/pkg/storage"
type blobInfo struct {
ID storage.BlobID `goja:"id"`
Bucket string `goja:"bucket"`
ModTime int64 `goja:"modTime"`
Size int64 `goja:"size"`
ContentType string `goja:"contentType"`
}
func toGojaBlobInfo(blob storage.BlobInfo) blobInfo {
return blobInfo{
ID: blob.ID(),
Bucket: blob.Bucket(),
ModTime: blob.ModTime().Unix(),
Size: blob.Size(),
ContentType: blob.ContentType(),
}
}

View File

@ -1,4 +1,4 @@
package module
package blob
import (
"context"

499
pkg/module/blob/module.go Normal file
View File

@ -0,0 +1,499 @@
package blob
import (
"context"
"fmt"
"io"
"mime/multipart"
"os"
"sort"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"forge.cadoles.com/arcad/edge/pkg/module/util"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/dop251/goja"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
const (
DefaultBlobBucket string = "default"
)
type Module struct {
server *app.Server
bus bus.Bus
store storage.BlobStore
}
func (m *Module) Name() string {
return "blob"
}
func (m *Module) Export(export *goja.Object) {
funcs := map[string]any{
"listBuckets": m.listBuckets,
"deleteBucket": m.deleteBucket,
"getBucketSize": m.getBucketSize,
"listBlobs": m.listBlobs,
"getBlobInfo": m.getBlobInfo,
"readBlob": m.readBlob,
"writeBlob": m.writeBlob,
"deleteBlob": m.deleteBlob,
}
for name, fn := range funcs {
if err := export.Set(name, fn); err != nil {
panic(errors.Wrapf(err, "could not set '%s' function", name))
}
}
if err := export.Set("DEFAULT_BUCKET", DefaultBlobBucket); err != nil {
panic(errors.Wrap(err, "could not set 'DEFAULT_BUCKET' property"))
}
}
func (m *Module) listBuckets(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
buckets, err := m.store.ListBuckets(ctx)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
defaultBucketIndex := sort.SearchStrings(buckets, DefaultBlobBucket)
if defaultBucketIndex == 0 {
buckets = append(buckets, DefaultBlobBucket)
} else {
buckets[defaultBucketIndex] = DefaultBlobBucket
}
return rt.ToValue(buckets)
}
func (m *Module) writeBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
blobID := assertBlobID(call.Argument(2), rt)
rawData := call.Argument(3).Export()
var data []byte
switch typ := rawData.(type) {
case []byte:
data = typ
case string:
data = []byte(typ)
default:
data = []byte(fmt.Sprintf("%v", typ))
}
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
}
}()
writer, err := bucket.NewWriter(ctx, blobID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
defer func() {
if err := writer.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
logger.Error(ctx, "could not close blob writer", logger.E(errors.WithStack(err)))
}
}()
if _, err := writer.Write(data); err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return nil
}
func (m *Module) getBlobInfo(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
blobID := assertBlobID(call.Argument(2), rt)
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
}
}()
blobInfo, err := bucket.Get(ctx, blobID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(toGojaBlobInfo(blobInfo))
}
func (m *Module) readBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
blobID := assertBlobID(call.Argument(2), rt)
reader, _, err := m.openBlob(ctx, bucketName, blobID)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
defer func() {
if err := reader.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
logger.Error(ctx, "could not close blob reader", logger.E(errors.WithStack(err)))
}
}()
data, err := io.ReadAll(reader)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(rt.NewArrayBuffer(data))
}
func (m *Module) deleteBlob(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
blobID := assertBlobID(call.Argument(2), rt)
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
if err := bucket.Delete(ctx, blobID); err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return nil
}
func (m *Module) listBlobs(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
blobInfos, err := bucket.List(ctx)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
gojaBlobInfos := make([]blobInfo, len(blobInfos))
for i, b := range blobInfos {
gojaBlobInfos[i] = toGojaBlobInfo(b)
}
return rt.ToValue(gojaBlobInfos)
}
func (m *Module) deleteBucket(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
if err := m.store.DeleteBucket(ctx, bucketName); err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return nil
}
func (m *Module) getBucketSize(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx := util.AssertContext(call.Argument(0), rt)
bucketName := util.AssertString(call.Argument(1), rt)
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
size, err := bucket.Size(ctx)
if err != nil {
panic(rt.ToValue(errors.WithStack(err)))
}
return rt.ToValue(size)
}
func (m *Module) handleMessages() {
ctx := context.Background()
go func() {
err := m.bus.Reply(ctx, MessageNamespaceUploadRequest, func(msg bus.Message) (bus.Message, error) {
uploadRequest, ok := msg.(*MessageUploadRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message upload request, got '%T'", msg)
}
res, err := m.handleUploadRequest(uploadRequest)
if err != nil {
logger.Error(ctx, "could not handle upload request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
logger.Debug(ctx, "upload request response", logger.F("response", res))
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}()
err := m.bus.Reply(ctx, MessageNamespaceDownloadRequest, func(msg bus.Message) (bus.Message, error) {
downloadRequest, ok := msg.(*MessageDownloadRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message download request, got '%T'", msg)
}
res, err := m.handleDownloadRequest(downloadRequest)
if err != nil {
logger.Error(ctx, "could not handle download request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}
func (m *Module) handleUploadRequest(req *MessageUploadRequest) (*MessageUploadResponse, error) {
blobID := storage.NewBlobID()
res := NewMessageUploadResponse(req.RequestID)
ctx := logger.With(req.Context, logger.F("blobID", blobID))
blobInfo := map[string]interface{}{
"size": req.FileHeader.Size,
"filename": req.FileHeader.Filename,
"contentType": req.FileHeader.Header.Get("Content-Type"),
}
rawResult, err := m.server.ExecFuncByName(ctx, "onBlobUpload", ctx, blobID, blobInfo, req.Metadata)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onBlobUpload result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
if res.Allow {
bucket := DefaultBlobBucket
rawBucket, exists := result["bucket"]
if exists {
bucket, ok = rawBucket.(string)
if !ok {
return nil, errors.Errorf("invalid 'bucket' result property: got type '%T', expected type '%T'", bucket, "")
}
}
if err := m.saveBlob(ctx, bucket, blobID, *req.FileHeader); err != nil {
return nil, errors.WithStack(err)
}
res.Bucket = bucket
res.BlobID = blobID
}
return res, nil
}
func (m *Module) saveBlob(ctx context.Context, bucketName string, blobID storage.BlobID, fileHeader multipart.FileHeader) error {
file, err := fileHeader.Open()
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
}
}()
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)))
}
}()
writer, err := bucket.NewWriter(ctx, blobID)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := file.Close(); err != nil {
logger.Error(ctx, "could not close file", logger.E(errors.WithStack(err)))
}
}()
defer func() {
if err := writer.Close(); err != nil {
logger.Error(ctx, "could not close writer", logger.E(errors.WithStack(err)))
}
}()
if _, err := io.Copy(writer, file); err != nil {
return errors.WithStack(err)
}
return nil
}
func (m *Module) handleDownloadRequest(req *MessageDownloadRequest) (*MessageDownloadResponse, error) {
res := NewMessageDownloadResponse(req.RequestID)
rawResult, err := m.server.ExecFuncByName(req.Context, "onBlobDownload", req.Context, req.Bucket, req.BlobID)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onBlobDownload result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
reader, info, err := m.openBlob(req.Context, req.Bucket, req.BlobID)
if err != nil && !errors.Is(err, storage.ErrBlobNotFound) {
return nil, errors.WithStack(err)
}
if reader != nil {
res.Blob = reader
}
if info != nil {
res.BlobInfo = info
}
return res, nil
}
func (m *Module) openBlob(ctx context.Context, bucketName string, blobID storage.BlobID) (io.ReadSeekCloser, storage.BlobInfo, error) {
bucket, err := m.store.OpenBucket(ctx, bucketName)
if err != nil {
return nil, nil, errors.WithStack(err)
}
defer func() {
if err := bucket.Close(); err != nil {
logger.Error(ctx, "could not close bucket", logger.E(errors.WithStack(err)), logger.F("bucket", bucket))
}
}()
info, err := bucket.Get(ctx, blobID)
if err != nil {
return nil, nil, errors.WithStack(err)
}
reader, err := bucket.NewReader(ctx, blobID)
if err != nil {
return nil, nil, errors.WithStack(err)
}
return reader, info, nil
}
func ModuleFactory(bus bus.Bus, store storage.BlobStore) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
mod := &Module{
store: store,
bus: bus,
server: server,
}
go mod.handleMessages()
return mod
}
}
func assertBlobID(value goja.Value, rt *goja.Runtime) storage.BlobID {
blobID, ok := value.Export().(storage.BlobID)
if !ok {
rawBlobID, ok := value.Export().(string)
if !ok {
panic(rt.NewTypeError(fmt.Sprintf("blob id must be a blob or a string, got '%T'", value.Export())))
}
blobID = storage.BlobID(rawBlobID)
}
return blobID
}

View File

@ -0,0 +1,44 @@
package blob
import (
"io/ioutil"
"testing"
"cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestBlobModule(t *testing.T) {
t.Parallel()
logger.SetLevel(slog.LevelDebug)
bus := memory.NewBus()
store := sqlite.NewBlobStore(":memory:?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000")
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
ModuleFactory(bus, store),
)
data, err := ioutil.ReadFile("testdata/blob.js")
if err != nil {
t.Fatal(err)
}
if err := server.Load("testdata/blob.js", string(data)); err != nil {
t.Fatal(err)
}
defer server.Stop()
if err := server.Start(); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
}

79
pkg/module/blob/testdata/blob.js vendored Normal file
View File

@ -0,0 +1,79 @@
var ctx = context.new();
var buckets = blob.listBuckets(ctx);
if (!buckets || buckets.length === 0) {
throw new Error("buckets should not be empty");
}
var size = blob.getBucketSize(ctx, blob.DEFAULT_BUCKET);
if (size !== 0) {
throw new Error("bucket size: expected '0', got '"+size+"'");
}
var newBucket = "mybucket"
var blobId = "foo"
var data = (new Date()).toString();
blob.writeBlob(ctx, newBucket, blobId, data)
buckets = blob.listBuckets(ctx);
if (buckets.length !== 2) {
throw new Error("buckets.length: expected '2', got '"+buckets.length+"'");
}
size = blob.getBucketSize(ctx, newBucket);
if (size !== data.length) {
throw new Error("bucket size: expected '"+data.length+"', got '"+size+"'");
}
var blobInfos = blob.listBlobs(ctx, newBucket);
if (blobInfos.length !== 1) {
throw new Error("blobInfos.length: expected '1', got '"+blobInfos.length+"'");
}
if (blobInfos[0].id != blobId) {
throw new Error("blobInfos[0].id: expected '"+blobId+"', got '"+blobInfos[0].id+"'");
}
if (blobInfos[0].contentType != "text/plain; charset=utf-8") {
throw new Error("blobInfos[0].contentType: expected 'text/plain; charset=utf-8', got '"+blobInfos[0].contentType+"'");
}
if (blobInfos[0].size != data.length) {
throw new Error("blobInfos[0].size: expected '"+data.length+"', got '"+blobInfos[0].size+"'");
}
var readData = blob.readBlob(ctx, newBucket, blobId)
if (!readData) {
throw new Error("readData should not be nil");
}
var buckets = blob.listBuckets(ctx);
if (!buckets || buckets.length !== 2) {
throw new Error("buckets.length should be 2");
}
blob.deleteBlob(ctx, newBucket, blobId)
blobInfos = blob.listBlobs(ctx, newBucket);
console.log(blobInfos);
if (blobInfos.length !== 0) {
throw new Error("blobInfos.length: expected '0', got '"+blobInfos.length+"'");
}
blob.deleteBucket(ctx, newBucket)
buckets = blob.listBuckets(ctx);
if (buckets.length !== 1) {
throw new Error("buckets.length: expected '1', got '"+buckets.length+"'");
}

View File

@ -39,7 +39,7 @@ const (
)
func getDeviceClientByUUID(ctx context.Context, uuid string) (*cast.Client, error) {
device, err := findDeviceByUUID(ctx, uuid)
device, err := FindDeviceByUUID(ctx, uuid)
if err != nil {
return nil, errors.WithStack(err)
}
@ -49,7 +49,7 @@ func getDeviceClientByUUID(ctx context.Context, uuid string) (*cast.Client, erro
return client, nil
}
func findDeviceByUUID(ctx context.Context, uuid string) (*Device, error) {
func FindDeviceByUUID(ctx context.Context, uuid string) (*Device, error) {
service := discovery.NewService(ctx)
defer service.Stop()
@ -83,7 +83,7 @@ LOOP:
return nil, errors.WithStack(ErrDeviceNotFound)
}
func findDevices(ctx context.Context) ([]*Device, error) {
func FindDevices(ctx context.Context) ([]*Device, error) {
service := discovery.NewService(ctx)
defer service.Stop()
@ -124,7 +124,7 @@ LOOP:
return devices, nil
}
func loadURL(ctx context.Context, deviceUUID string, url string) error {
func LoadURL(ctx context.Context, deviceUUID string, url string) error {
client, err := getDeviceClientByUUID(ctx, deviceUUID)
if err != nil {
return errors.WithStack(err)
@ -153,7 +153,7 @@ func isLoadURLContextExceeded(err error) bool {
return err.Error() == "Failed to send load command: context deadline exceeded"
}
func stopCast(ctx context.Context, deviceUUID string) error {
func StopCast(ctx context.Context, deviceUUID string) error {
client, err := getDeviceClientByUUID(ctx, deviceUUID)
if err != nil {
return errors.WithStack(err)

View File

@ -26,7 +26,7 @@ func TestCastLoadURL(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
devices, err := findDevices(ctx)
devices, err := FindDevices(ctx)
if err != nil {
t.Error(errors.WithStack(err))
}
@ -40,7 +40,7 @@ func TestCastLoadURL(t *testing.T) {
ctx, cancel2 := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel2()
if err := loadURL(ctx, dev.UUID, "https://go.dev"); err != nil {
if err := LoadURL(ctx, dev.UUID, "https://go.dev"); err != nil {
t.Error(errors.WithStack(err))
}
@ -57,7 +57,7 @@ func TestCastLoadURL(t *testing.T) {
ctx, cancel4 := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel4()
if err := stopCast(ctx, dev.UUID); err != nil {
if err := StopCast(ctx, dev.UUID); err != nil {
t.Error(errors.WithStack(err))
}
}

View File

@ -72,7 +72,7 @@ func (m *Module) refreshDevices(call goja.FunctionCall, rt *goja.Runtime) goja.V
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
devices, err := findDevices(ctx)
devices, err := FindDevices(ctx)
if err != nil && !errors.Is(err, context.DeadlineExceeded) {
err = errors.WithStack(err)
logger.Error(ctx, "error refreshing casting devices list", logger.E(errors.WithStack(err)))
@ -128,7 +128,7 @@ func (m *Module) loadUrl(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
err := loadURL(ctx, deviceUUID, url)
err := LoadURL(ctx, deviceUUID, url)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error while casting url", logger.E(err))
@ -166,7 +166,7 @@ func (m *Module) stopCast(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
err := stopCast(ctx, deviceUUID)
err := StopCast(ctx, deviceUUID)
if err != nil {
err = errors.WithStack(err)
logger.Error(ctx, "error while quitting casting device app", logger.E(errors.WithStack(err)))

View File

@ -0,0 +1,49 @@
package fetch
import (
"context"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/bus"
"github.com/oklog/ulid/v2"
)
const (
MessageNamespaceFetchRequest bus.MessageNamespace = "fetchRequest"
MessageNamespaceFetchResponse bus.MessageNamespace = "fetchResponse"
)
type MessageFetchRequest struct {
Context context.Context
RequestID string
URL *url.URL
RemoteAddr string
}
func (m *MessageFetchRequest) MessageNamespace() bus.MessageNamespace {
return MessageNamespaceFetchRequest
}
func NewMessageFetchRequest(ctx context.Context, remoteAddr string, url *url.URL) *MessageFetchRequest {
return &MessageFetchRequest{
Context: ctx,
RequestID: ulid.Make().String(),
RemoteAddr: remoteAddr,
URL: url,
}
}
type MessageFetchResponse struct {
RequestID string
Allow bool
}
func (m *MessageFetchResponse) MessageNamespace() bus.MessageNamespace {
return MessageNamespaceFetchResponse
}
func NewMessageFetchResponse(requestID string) *MessageFetchResponse {
return &MessageFetchResponse{
RequestID: requestID,
}
}

122
pkg/module/fetch/module.go Normal file
View File

@ -0,0 +1,122 @@
package fetch
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"github.com/dop251/goja"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type Module struct {
server *app.Server
bus bus.Bus
}
func (m *Module) Name() string {
return "fetch"
}
func (m *Module) Export(export *goja.Object) {
funcs := map[string]any{
"get": m.get,
}
for name, fn := range funcs {
if err := export.Set(name, fn); err != nil {
panic(errors.Wrapf(err, "could not set '%s' function", name))
}
}
}
func (m *Module) get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
// ctx := util.AssertContext(call.Argument(0), rt)
return nil
}
func (m *Module) handleMessages() {
ctx := context.Background()
err := m.bus.Reply(ctx, MessageNamespaceFetchRequest, func(msg bus.Message) (bus.Message, error) {
fetchRequest, ok := msg.(*MessageFetchRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message fetch request, got '%T'", msg)
}
res, err := m.handleFetchRequest(fetchRequest)
if err != nil {
logger.Error(ctx, "could not handle fetch request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
logger.Debug(ctx, "fetch request response", logger.F("response", res))
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}
func (m *Module) handleFetchRequest(req *MessageFetchRequest) (*MessageFetchResponse, error) {
res := NewMessageFetchResponse(req.RequestID)
ctx := logger.With(
req.Context,
logger.F("url", req.URL.String()),
logger.F("remoteAddr", req.RemoteAddr),
logger.F("requestID", req.RequestID),
)
rawResult, err := m.server.ExecFuncByName(ctx, "onClientFetch", ctx, req.URL.String(), req.RemoteAddr)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onClientFetch result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
return res, nil
}
func ModuleFactory(bus bus.Bus) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
mod := &Module{
bus: bus,
server: server,
}
go mod.handleMessages()
return mod
}
}

View File

@ -0,0 +1,84 @@
package fetch
import (
"context"
"io/ioutil"
"net/url"
"testing"
"time"
"cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/arcad/edge/pkg/module"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestFetchModule(t *testing.T) {
t.Parallel()
logger.SetLevel(slog.LevelDebug)
bus := memory.NewBus()
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
ModuleFactory(bus),
)
data, err := ioutil.ReadFile("testdata/fetch.js")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Load("testdata/fetch.js", string(data)); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer server.Stop()
if err := server.Start(); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
// Wait for module to startup
time.Sleep(1 * time.Second)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
remoteAddr := "127.0.0.1"
url, _ := url.Parse("http://example.com")
rawReply, err := bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
reply, ok := rawReply.(*MessageFetchResponse)
if !ok {
t.Fatalf("unexpected reply type '%T'", rawReply)
}
if e, g := true, reply.Allow; e != g {
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
}
url, _ = url.Parse("https://google.com")
rawReply, err = bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
reply, ok = rawReply.(*MessageFetchResponse)
if !ok {
t.Fatalf("unexpected reply type '%T'", rawReply)
}
if e, g := false, reply.Allow; e != g {
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
}
}

7
pkg/module/fetch/testdata/fetch.js vendored Normal file
View File

@ -0,0 +1,7 @@
var ctx = context.new();
function onClientFetch(ctx, url, remoteAddr) {
if (url === 'http://example.com') return { allow: true };
return { allow: false };
}

View File

@ -15,7 +15,7 @@ import (
func TestStoreModule(t *testing.T) {
logger.SetLevel(logger.LevelDebug)
store := sqlite.NewDocumentStore(":memory:")
store := sqlite.NewDocumentStore(":memory:?_pragma=foreign_keys(1)&_pragma=busy_timeout=60000")
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),

View File

@ -12,7 +12,7 @@ func AssertType[T any](v goja.Value, rt *goja.Runtime) T {
return c
}
panic(rt.ToValue(errors.Errorf("expected value to be a '%T', got '%T'", *new(T), v.Export())))
panic(rt.ToValue(errors.Errorf("expected value to be a '%T', got '%T'", new(T), v.Export())))
}
func AssertContext(v goja.Value, r *goja.Runtime) context.Context {

View File

@ -3785,7 +3785,8 @@ var Edge = (() => {
// pkg/sdk/client/src/index.ts
var src_exports = {};
__export(src_exports, {
default: () => src_default
client: () => client,
crossFrameMessenger: () => crossFrameMessenger
});
// pkg/sdk/client/src/event-target.ts
@ -3864,6 +3865,8 @@ var Edge = (() => {
var import_sockjs_client = __toESM(require_entry());
var EventTypeMessage = "message";
var EdgeAuth = "edge-auth";
var EdgeAuthTokenRequest = "edge_auth_token_request";
var EdgeAuthTokenResponse = "edge_auth_token_reponse";
var Client = class extends EventTarget {
constructor(autoReconnect = true) {
super();
@ -3871,6 +3874,7 @@ var Edge = (() => {
this._onConnectionClose = this._onConnectionClose.bind(this);
this._onConnectionMessage = this._onConnectionMessage.bind(this);
this._handleRPCResponse = this._handleRPCResponse.bind(this);
this._handleEdgeAuthTokenRequest = this._handleEdgeAuthTokenRequest.bind(this);
this._rpcID = 0;
this._pendingRPC = {};
this._queue = [];
@ -3883,12 +3887,22 @@ var Edge = (() => {
this.send = this.send.bind(this);
this.upload = this.upload.bind(this);
this.addEventListener(EventTypeMessage, this._handleRPCResponse);
window.addEventListener("message", this._handleEdgeAuthTokenRequest);
}
connect(token = "") {
let getToken;
if (token) {
getToken = Promise.resolve(token);
} else {
getToken = this._retrieveToken();
}
return getToken.then((token2) => this._connect(token2));
}
disconnect() {
this._cleanupConnection();
}
_connect(token) {
return new Promise((resolve, reject) => {
if (token == "") {
token = this._getAuthCookieToken();
}
const url = `//${document.location.host}/edge/sock?${EdgeAuth}=${token}`;
this._log("opening connection to", url);
const conn = new import_sockjs_client.default(url);
@ -3919,15 +3933,64 @@ var Edge = (() => {
conn.addEventListener("close", onError);
});
}
disconnect() {
this._cleanupConnection();
_retrieveToken() {
let token = this._getAuthCookieToken();
if (token) {
return Promise.resolve(token);
}
return this._getParentFrameToken();
;
}
_getAuthCookieToken() {
const cookie = document.cookie.split("; ").find((row) => row.startsWith(EdgeAuth));
let token = "";
if (cookie) {
return cookie.split("=")[1];
token = cookie.split("=")[1];
}
return "";
return token;
}
_getParentFrameToken(timeout = 5e3) {
if (!window.parent || window.parent === window) {
return Promise.resolve("");
}
return new Promise((resolve, reject) => {
let timedOut = false;
const timeoutId = setTimeout(() => {
timedOut = true;
reject(new Error("Edge auth token request timed out !"));
}, timeout);
const listener = (evt) => {
const message = evt.data;
if (!message || !message.type || !message.data) {
return;
}
if (message.type !== EdgeAuthTokenResponse) {
return;
}
window.parent.removeEventListener("message", listener);
clearTimeout(timeoutId);
if (timedOut)
return;
if (!message.data || !message.data.token) {
reject("Unexpected auth token request response !");
return;
}
resolve(message.data.token);
};
window.parent.addEventListener("message", listener);
window.parent.postMessage({ type: EdgeAuthTokenRequest }, "*");
});
}
_handleEdgeAuthTokenRequest(evt) {
const message = evt.data;
if (!message || !message.type || message.type !== EdgeAuthTokenRequest) {
return;
}
if (!evt.source) {
return;
}
const token = this._getAuthCookieToken();
evt.source.postMessage({ type: EdgeAuthTokenResponse, data: { token } });
}
_onConnectionMessage(evt) {
const rawMessage = JSON.parse(evt.data);
@ -4084,11 +4147,76 @@ var Edge = (() => {
blobUrl(bucket, blobId) {
return `/edge/api/v1/download/${bucket}/${blobId}`;
}
externalUrl(url) {
return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`;
}
};
// pkg/sdk/client/src/crossframe-messenger.ts
var CrossFrameMessenger = class extends EventTarget {
constructor() {
super();
this.debug = false;
this._handleWindowMessage = this._handleWindowMessage.bind(this);
this._initObservers = this._initObservers.bind(this);
window.addEventListener("load", this._initObservers);
window.addEventListener("message", this._handleWindowMessage);
}
post(message, target = window.parent) {
if (!target)
return;
this._log("sending crossframe message", message);
target.postMessage(message, "*");
}
_log(...args) {
if (!this.debug)
return;
console.log(...args);
}
_handleWindowMessage(evt) {
const message = evt.data;
if (!message || !message.type || !message.data) {
return;
}
const event = new CustomEvent(message.type, {
cancelable: true,
detail: message.data
});
this.dispatchEvent(event);
}
_initObservers() {
this._initResizeObserver();
this._initTitleMutationObserver();
}
_initTitleMutationObserver() {
const titleObserver = new MutationObserver((mutations) => {
const title2 = mutations[0].target.textContent;
this.post({ type: "title_changed" /* TITLE_CHANGED */, data: { title: title2 } });
});
const title = document.querySelector("title");
if (!title)
return;
this.post({ type: "title_changed" /* TITLE_CHANGED */, data: { title: title.textContent } });
titleObserver.observe(title, { subtree: true, characterData: true, childList: true });
}
_initResizeObserver() {
const resizeObserver = new ResizeObserver(() => {
const rect = document.documentElement.getBoundingClientRect();
const height = rect.height;
const width = rect.width;
this.post({ type: "size_changed" /* SIZE_CHANGED */, data: { height, width } });
});
const body = document.body;
if (!body)
return;
resizeObserver.observe(document.documentElement);
}
};
// pkg/sdk/client/src/index.ts
var src_default = new Client();
var client = new Client();
var crossFrameMessenger = new CrossFrameMessenger();
return __toCommonJS(src_exports);
})();
Edge=Edge.default;
EdgeFrame=Edge.crossFrameMessenger;Edge=Edge.client
//# sourceMappingURL=client.js.map

File diff suppressed because one or more lines are too long

View File

@ -5,6 +5,8 @@ import SockJS from 'sockjs-client';
const EventTypeMessage = "message";
const EdgeAuth = "edge-auth"
const EdgeAuthTokenRequest = "edge_auth_token_request"
const EdgeAuthTokenResponse = "edge_auth_token_reponse"
export class Client extends EventTarget {
@ -19,80 +21,158 @@ export class Client extends EventTarget {
constructor(autoReconnect = true) {
super();
this._conn = null;
this._onConnectionClose = this._onConnectionClose.bind(this);
this._onConnectionMessage = this._onConnectionMessage.bind(this);
this._handleRPCResponse = this._handleRPCResponse.bind(this);
this._handleEdgeAuthTokenRequest = this._handleEdgeAuthTokenRequest.bind(this);
this._rpcID = 0;
this._pendingRPC = {};
this._queue = [];
this._reconnectionDelay = 250;
this._autoReconnect = autoReconnect;
this.debug = false;
this.connect = this.connect.bind(this);
this.disconnect = this.disconnect.bind(this);
this.rpc = this.rpc.bind(this);
this.send = this.send.bind(this);
this.upload = this.upload.bind(this);
this.addEventListener(EventTypeMessage, this._handleRPCResponse);
window.addEventListener('message', this._handleEdgeAuthTokenRequest);
}
connect(token = "") {
return new Promise((resolve, reject) => {
if (token == "") {
token = this._getAuthCookieToken()
}
connect(token = ""): Promise<Client> {
let getToken: Promise<string>
const url = `//${document.location.host}/edge/sock?${EdgeAuth}=${token}`;
this._log("opening connection to", url);
const conn: any = new SockJS(url);
if (token) {
getToken = Promise.resolve(token)
} else {
getToken = this._retrieveToken()
}
const onOpen = () => {
this._log('client connected');
resetHandlers();
conn.onclose = this._onConnectionClose;
conn.onmessage = this._onConnectionMessage;
this._conn = conn;
this._sendQueued();
setTimeout(() => {
this._dispatchConnect();
}, 0);
return resolve(this);
};
const onError = (evt) => {
resetHandlers();
this._scheduleReconnection();
return reject(evt);
};
const resetHandlers = () => {
conn.removeEventListener('open', onOpen);
conn.removeEventListener('close', onError);
conn.removeEventListener('error', onError);
};
conn.addEventListener('open', onOpen);
conn.addEventListener('error', onError);
conn.addEventListener('close', onError);
});
return getToken.then(token => this._connect(token))
}
disconnect() {
this._cleanupConnection();
}
_getAuthCookieToken() {
const cookie = document.cookie.split("; ")
.find((row) => row.startsWith(EdgeAuth));
if (cookie) {
return cookie.split("=")[1];
_connect(token: string): Promise<Client> {
return new Promise((resolve, reject) => {
const url = `//${document.location.host}/edge/sock?${EdgeAuth}=${token}`;
this._log("opening connection to", url);
const conn: any = new SockJS(url);
const onOpen = () => {
this._log('client connected');
resetHandlers();
conn.onclose = this._onConnectionClose;
conn.onmessage = this._onConnectionMessage;
this._conn = conn;
this._sendQueued();
setTimeout(() => {
this._dispatchConnect();
}, 0);
return resolve(this);
};
const onError = (evt) => {
resetHandlers();
this._scheduleReconnection();
return reject(evt);
};
const resetHandlers = () => {
conn.removeEventListener('open', onOpen);
conn.removeEventListener('close', onError);
conn.removeEventListener('error', onError);
};
conn.addEventListener('open', onOpen);
conn.addEventListener('error', onError);
conn.addEventListener('close', onError);
})
}
_retrieveToken(): Promise<string> {
let token = this._getAuthCookieToken();
if (token) {
return Promise.resolve(token);
}
return "";
return this._getParentFrameToken();;
}
_getAuthCookieToken(): string {
const cookie = document.cookie.split("; ")
.find((row) => row.startsWith(EdgeAuth));
let token = "";
if (cookie) {
token = cookie.split("=")[1];
}
return token;
}
_getParentFrameToken(timeout = 5000): Promise<string> {
if (!window.parent || window.parent === window) {
return Promise.resolve("");
}
return new Promise((resolve, reject) => {
let timedOut = false;
const timeoutId = setTimeout(() => {
timedOut = true;
reject(new Error("Edge auth token request timed out !"));
}, timeout);
const listener = (evt) => {
const message = evt.data;
if (!message || !message.type || !message.data) {
return
}
if (message.type !== EdgeAuthTokenResponse) {
return;
}
window.parent.removeEventListener('message', listener);
clearTimeout(timeoutId);
if (timedOut) return;
if (!message.data || !message.data.token) {
reject("Unexpected auth token request response !");
return;
}
resolve(message.data.token);
}
window.parent.addEventListener('message', listener);
window.parent.postMessage({ type: EdgeAuthTokenRequest }, '*');
})
}
_handleEdgeAuthTokenRequest(evt: MessageEvent) {
const message = evt.data;
if (!message || !message.type || message.type !== EdgeAuthTokenRequest) {
return;
}
if (!evt.source) {
return;
}
const token = this._getAuthCookieToken();
evt.source.postMessage({ type: EdgeAuthTokenResponse, data: { token }});
}
_onConnectionMessage(evt) {
@ -107,7 +187,7 @@ export class Client extends EventTarget {
_handleRPCResponse(evt) {
const { jsonrpc, id, error, result } = evt.detail;
if (jsonrpc !== '2.0' || id === undefined) return;
if (!evt.detail.hasOwnProperty("error") && !evt.detail.hasOwnProperty("result")) return;
@ -215,20 +295,20 @@ export class Client extends EventTarget {
return this._conn !== null;
}
upload(blob: string|Blob, metadata: any) {
upload(blob: string | Blob, metadata: any) {
return new Promise((resolve, reject) => {
const formData = new FormData();
formData.set("file", blob);
if (metadata) {
try {
formData.set("metadata", JSON.stringify(metadata));
} catch(err) {
} catch (err) {
return reject(err);
}
}
const xhr = new XMLHttpRequest();
const result = {
onProgress: null,
abort: () => xhr.abort(),
@ -238,7 +318,7 @@ export class Client extends EventTarget {
let data;
try {
data = JSON.parse(xhr.responseText);
} catch(err) {
} catch (err) {
reject(err);
return;
}
@ -264,7 +344,11 @@ export class Client extends EventTarget {
});
}
blobUrl(bucket: string, blobId: string) {
blobUrl(bucket: string, blobId: string): string {
return `/edge/api/v1/download/${bucket}/${blobId}`;
}
externalUrl(url: string): string {
return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`
}
}

View File

@ -0,0 +1,87 @@
import { EventTarget } from "./event-target";
enum CrossFrameMessageType {
SIZE_CHANGED = "size_changed",
TITLE_CHANGED = "title_changed"
}
interface CrossFrameMessage {
type: CrossFrameMessageType
data: { [key: string]: any }
}
export class CrossFrameMessenger extends EventTarget {
debug: boolean;
constructor() {
super()
this.debug = false;
this._handleWindowMessage = this._handleWindowMessage.bind(this);
this._initObservers = this._initObservers.bind(this);
window.addEventListener('load', this._initObservers);
window.addEventListener('message', this._handleWindowMessage)
}
post(message: CrossFrameMessage, target: Window = window.parent) {
if (!target) return;
this._log("sending crossframe message", message);
target.postMessage(message, '*');
}
_log(...args) {
if (!this.debug) return;
console.log(...args);
}
_handleWindowMessage(evt: MessageEvent) {
const message = evt.data;
if (!message || !message.type || !message.data) {
return;
}
const event = new CustomEvent(message.type, {
cancelable: true,
detail: message.data
});
this.dispatchEvent(event);
}
_initObservers() {
this._initResizeObserver();
this._initTitleMutationObserver();
}
_initTitleMutationObserver() {
const titleObserver = new MutationObserver((mutations) => {
const title = mutations[0].target.textContent;
this.post({ type: CrossFrameMessageType.TITLE_CHANGED, data: { title }});
});
const title = document.querySelector('title');
if (!title) return;
this.post({ type: CrossFrameMessageType.TITLE_CHANGED, data: { title: title.textContent }});
titleObserver.observe(title, { subtree: true, characterData: true, childList: true });
}
_initResizeObserver() {
const resizeObserver = new ResizeObserver(() => {
const rect = document.documentElement.getBoundingClientRect();
const height = rect.height;
const width = rect.width;
this.post({ type: CrossFrameMessageType.SIZE_CHANGED, data: { height, width }});
});
const body = document.body;
if (!body) return;
resizeObserver.observe(document.documentElement);
}
}

View File

@ -1,3 +1,5 @@
import { Client } from './client.js';
import { CrossFrameMessenger } from './crossframe-messenger.js';
export default new Client();
export const client = new Client();
export const crossFrameMessenger = new CrossFrameMessenger();

View File

@ -35,6 +35,10 @@ func (b *BlobBucket) Size(ctx context.Context) (int64, error) {
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
size = nullSize.Int64
return nil
@ -68,8 +72,11 @@ func (b *BlobBucket) Close() error {
func (b *BlobBucket) Delete(ctx context.Context, id storage.BlobID) error {
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `DELETE FROM blobs WHERE bucket = $1 AND id = $2`
args := []any{b.name, id}
if _, err := tx.ExecContext(ctx, query, b.name, id); err != nil {
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return errors.WithStack(err)
}
@ -88,7 +95,11 @@ func (b *BlobBucket) Get(ctx context.Context, id storage.BlobID) (storage.BlobIn
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT content_type, mod_time, size FROM blobs WHERE bucket = $1 AND id = $2`
row := tx.QueryRowContext(ctx, query, b.name, id)
args := []any{b.name, id}
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
row := tx.QueryRowContext(ctx, query, args...)
var (
contentType string
@ -104,6 +115,10 @@ func (b *BlobBucket) Get(ctx context.Context, id storage.BlobID) (storage.BlobIn
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
blobInfo = &BlobInfo{
id: id,
bucket: b.name,
@ -127,12 +142,21 @@ func (b *BlobBucket) List(ctx context.Context) ([]storage.BlobInfo, error) {
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT id, content_type, mod_time, size FROM blobs WHERE bucket = $1`
args := []any{b.name}
rows, err := tx.QueryContext(ctx, query, b.name)
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
blobs = make([]storage.BlobInfo, 0)
for rows.Next() {
@ -229,7 +253,12 @@ type blobWriterCloser struct {
// Write implements io.WriteCloser
func (wbc *blobWriterCloser) Write(p []byte) (int, error) {
logger.Debug(context.Background(), "writing data to blob", logger.F("data", p))
logger.Debug(
context.Background(), "writing data to blob",
logger.F("size", len(p)),
logger.F("blobID", wbc.id),
logger.F("bucket", wbc.bucket),
)
n, err := wbc.buf.Write(p)
if err != nil {
@ -266,14 +295,20 @@ func (wbc *blobWriterCloser) Close() error {
mime := mimetype.Detect(data)
modTime := time.Now().UTC()
_, err := tx.Exec(
query,
args := []any{
wbc.bucket,
wbc.id,
data,
mime.String(),
modTime,
len(data),
}
logger.Debug(ctx, "executing query", logger.F("query", query))
_, err := tx.Exec(
query,
args...,
)
if err != nil {
return errors.WithStack(err)

View File

@ -36,7 +36,7 @@ func (s *BlobStore) ListBuckets(ctx context.Context) ([]string, error) {
buckets := make([]string, 0)
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT DISTINCT name FROM blobs`
query := `SELECT DISTINCT bucket FROM blobs`
rows, err := tx.QueryContext(ctx, query)
if err != nil {
return errors.WithStack(err)

View File

@ -1,8 +1,10 @@
package sqlite
import (
"fmt"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
@ -19,7 +21,8 @@ func TestBlobStore(t *testing.T) {
t.Fatalf("%+v", errors.WithStack(err))
}
store := NewBlobStore(file)
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewBlobStore(dsn)
testsuite.TestBlobStore(t, store)
}

View File

@ -5,7 +5,6 @@ import (
"database/sql"
"fmt"
"math"
"sync"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
@ -18,10 +17,7 @@ import (
)
type DocumentStore struct {
db *sql.DB
path string
openOnce sync.Once
mutex sync.RWMutex
getDB getDBFunc
}
// Delete implements storage.DocumentStore
@ -74,6 +70,10 @@ func (s *DocumentStore) Get(ctx context.Context, collection string, id storage.D
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
document = storage.Document(data)
document[storage.DocumentAttrID] = id
@ -160,7 +160,11 @@ func (s *DocumentStore) Query(ctx context.Context, collection string, filter *fi
return errors.WithStack(err)
}
defer rows.Close()
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
documents = make([]storage.Document, 0)
@ -238,6 +242,10 @@ func (s *DocumentStore) Upsert(ctx context.Context, collection string, document
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
upsertedDocument = storage.Document(data)
upsertedDocument[storage.DocumentAttrID] = id
@ -256,7 +264,7 @@ func (s *DocumentStore) Upsert(ctx context.Context, collection string, document
func (s *DocumentStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
var db *sql.DB
db, err := s.getDatabase(ctx)
db, err := s.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
@ -268,67 +276,7 @@ func (s *DocumentStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) e
return nil
}
func (s *DocumentStore) getDatabase(ctx context.Context) (*sql.DB, error) {
s.mutex.RLock()
if s.db != nil {
defer s.mutex.RUnlock()
var err error
s.openOnce.Do(func() {
if err = s.ensureTables(ctx, s.db); err != nil {
err = errors.WithStack(err)
return
}
})
if err != nil {
return nil, errors.WithStack(err)
}
return s.db, nil
}
s.mutex.RUnlock()
var (
db *sql.DB
err error
)
s.openOnce.Do(func() {
db, err = sql.Open("sqlite", s.path)
if err != nil {
err = errors.WithStack(err)
return
}
if err = s.ensureTables(ctx, db); err != nil {
err = errors.WithStack(err)
return
}
})
if err != nil {
return nil, errors.WithStack(err)
}
if db != nil {
s.mutex.Lock()
s.db = db
s.mutex.Unlock()
}
s.mutex.RLock()
defer s.mutex.RUnlock()
return s.db, nil
}
func (s *DocumentStore) ensureTables(ctx context.Context, db *sql.DB) error {
func ensureTables(ctx context.Context, db *sql.DB) error {
err := withTx(ctx, db, func(tx *sql.Tx) error {
query := `
CREATE TABLE IF NOT EXISTS documents (
@ -396,18 +344,18 @@ func withLimitOffsetClause(query string, args []any, limit int, offset int) (str
}
func NewDocumentStore(path string) *DocumentStore {
getDB := newGetDBFunc(path, ensureTables)
return &DocumentStore{
db: nil,
path: path,
openOnce: sync.Once{},
getDB: getDB,
}
}
func NewDocumentStoreWithDB(db *sql.DB) *DocumentStore {
getDB := newGetDBFuncFromDB(db, ensureTables)
return &DocumentStore{
db: db,
path: "",
openOnce: sync.Once{},
getDB: getDB,
}
}

View File

@ -1,8 +1,10 @@
package sqlite
import (
"fmt"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
@ -10,7 +12,7 @@ import (
)
func TestDocumentStore(t *testing.T) {
// t.Parallel()
t.Parallel()
logger.SetLevel(logger.LevelDebug)
file := "./testdata/documentstore_test.sqlite"
@ -19,7 +21,8 @@ func TestDocumentStore(t *testing.T) {
t.Fatalf("%+v", errors.WithStack(err))
}
store := NewDocumentStore(file)
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewDocumentStore(dsn)
testsuite.TestDocumentStore(t, store)
}

View File

@ -8,7 +8,9 @@ import (
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"modernc.org/sqlite"
_ "modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
)
func Open(path string) (*sql.DB, error) {
@ -23,7 +25,7 @@ func Open(path string) (*sql.DB, error) {
func withTx(ctx context.Context, db *sql.DB, fn func(tx *sql.Tx) error) error {
var tx *sql.Tx
tx, err := db.Begin()
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return errors.WithStack(err)
}
@ -38,8 +40,27 @@ func withTx(ctx context.Context, db *sql.DB, fn func(tx *sql.Tx) error) error {
}
}()
if err = fn(tx); err != nil {
return errors.WithStack(err)
for {
if err = fn(tx); err != nil {
var sqlErr *sqlite.Error
if errors.As(err, &sqlErr) {
if sqlErr.Code() == sqlite3.SQLITE_BUSY {
logger.Warn(ctx, "database busy, retrying transaction")
if err := ctx.Err(); err != nil {
logger.Error(ctx, "could not execute transaction", logger.E(errors.WithStack(err)))
return errors.WithStack(err)
}
continue
}
}
return errors.WithStack(err)
}
break
}
if err = tx.Commit(); err != nil {

View File

@ -1 +1 @@
/*.sqlite
/*.sqlite*

View File

@ -8,7 +8,7 @@ import (
func TestBlobStore(t *testing.T, store storage.BlobStore) {
t.Run("Ops", func(t *testing.T) {
// t.Parallel()
t.Parallel()
testBlobStoreOps(t, store)
})
}

View File

@ -8,7 +8,7 @@ import (
func TestDocumentStore(t *testing.T, store storage.DocumentStore) {
t.Run("Ops", func(t *testing.T) {
// t.Parallel()
t.Parallel()
testDocumentStoreOps(t, store)
})
}

View File

@ -437,6 +437,7 @@ func testDocumentStoreOps(t *testing.T, store storage.DocumentStore) {
for _, tc := range documentStoreOpsTestCases {
func(tc documentStoreOpsTestCase) {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
if err := tc.Run(context.Background(), store); err != nil {
t.Errorf("%+v", errors.WithStack(err))
}