feat(app,module): fetch basic module

This commit is contained in:
wpetit 2023-04-02 17:59:33 +02:00
parent d8ce2901d2
commit fbb27d6ea4
17 changed files with 472 additions and 3 deletions

View File

@ -21,6 +21,7 @@ import (
authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
"forge.cadoles.com/arcad/edge/pkg/module/blob"
"forge.cadoles.com/arcad/edge/pkg/module/cast"
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
"forge.cadoles.com/arcad/edge/pkg/module/net"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
@ -208,6 +209,7 @@ func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStor
},
manifest,
)),
fetch.ModuleFactory(bus),
}
}

View File

@ -51,6 +51,10 @@ Edge.connect().then(() => {
> `TODO`
### `Edge.externalUrl(url: string): string`
Retourne une URL "locale" permettant d'accéder à une ressource externe, en fonction de règles propres à l'application. Voir module [`fetch`](../server-api/fetch.md).
## Événements
### `"message"`

View File

@ -26,6 +26,7 @@ Listes des modules disponibles côté serveur.
- [`cast`](./cast.md)
- [`console`](./console.md)
- [`context`](./context.md)
- [`fetch`](./fetch.md)
- [`net`](./net.md)
- [`rpc`](./rpc.md)
- [`store`](./store.md)

View File

@ -0,0 +1,33 @@
# Module `fetch`
Ce module permet l'accès à des ressources distantes (sur Internet) depuis votre application.
## Fonctions de rappel
Pour permettre aux utilisateurs d'accéder à des ressources distantes, vous devez déclarer la fonction `onClientFetch(ctx: Context, url: string, remoteAddr: string)` dans le fichier `server/main.js` de votre application.
### `onClientFetch(ctx: Context, url: string, remoteAddr: string)`
#### Usage
**Côté client**
```js
// Création d'une URL "locale" permettant d'accéder à la ressource distante
var url = Edge.externalUrl("http://example.com")
// Vous pouvez utiliser l'URL comme attribut `src` d'une balise <img> par exemple
// ou effectuer une requête fetch() avec celle ci.
fetch(url).then(res => res.text()).then(content => console.log(content));
```
**Côté serveur**
```js
function onClientFetch(ctx, url, remoteAddr) {
// Autoriser la récupération de l'URL demandée ou non
// Dans cet exemple, seule l'URL externe 'http://example.com' est autorisée
// Les autres URLs recevront une erreur HTTP 403 - Forbidden
var authorized = url === "http://example.com"
return { allow: authorized };
}
```

View File

@ -26,6 +26,7 @@
<script src="/test/rpc-module.js"></script>
<script src="/test/file-module.js"></script>
<script src="/test/app-module.js"></script>
<script src="/test/fetch-module.js"></script>
<script class="mocha-exec">
mocha.run();
</script>

View File

@ -0,0 +1,33 @@
describe('Fetch Module', function () {
before(() => {
return Edge.connect();
});
after(() => {
Edge.disconnect();
});
it('should fetch an authorized external url', function () {
var externalUrl = Edge.externalUrl("http://example.com");
return fetch(externalUrl)
.then(res => {
chai.assert.equal(res.status, 200)
return res.text()
})
.then(content => {
chai.assert.include(content, '<h1>Example Domain</h1>')
})
});
it('should not fetch an unauthorized external url', function () {
var externalUrl = Edge.externalUrl("https://google.com");
return fetch(externalUrl)
.then(res => {
chai.assert.equal(res.status, 403)
})
});
});

View File

@ -97,4 +97,8 @@ function getApp(ctx, params) {
function getAppUrl(ctx, params) {
var appId = params.appId;
return app.getUrl(ctx, appId);
}
function onClientFetch(ctx, url, remoteAddr) {
return { allow: url === 'http://example.com' };
}

112
pkg/http/fetch.go Normal file
View File

@ -0,0 +1,112 @@
package http
import (
"io"
"net/http"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/module"
"forge.cadoles.com/arcad/edge/pkg/module/fetch"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
h.mutex.RLock()
defer h.mutex.RUnlock()
ctx := r.Context()
ctx = module.WithContext(ctx, map[module.ContextKey]any{
ContextKeyOriginRequest: r,
})
rawURL := r.URL.Query().Get("url")
url, err := url.Parse(rawURL)
if err != nil {
jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
return
}
requestMsg := fetch.NewMessageFetchRequest(ctx, r.RemoteAddr, url)
reply, err := h.bus.Request(ctx, requestMsg)
if err != nil {
logger.Error(ctx, "could not retrieve fetch request reply", logger.E(errors.WithStack(err)))
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
logger.Debug(ctx, "fetch reply", logger.F("reply", reply))
responseMsg, ok := reply.(*fetch.MessageFetchResponse)
if !ok {
logger.Error(
ctx, "unexpected fetch response message",
logger.F("message", reply),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
if !responseMsg.Allow {
jsonError(w, http.StatusForbidden, errorCodeForbidden)
return
}
proxyReq, err := http.NewRequest(http.MethodGet, url.String(), nil)
if err != nil {
logger.Error(
ctx, "could not create proxy request",
logger.E(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
proxyReq.Header.Add("X-Forwarded-From", r.RemoteAddr)
res, err := h.httpClient.Do(proxyReq)
if err != nil {
logger.Error(
ctx, "could not execute proxy request",
logger.E(errors.WithStack(err)),
)
jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
return
}
defer func() {
if err := res.Body.Close(); err != nil {
logger.Error(
ctx, "could not close response body",
logger.E(errors.WithStack(err)),
)
}
}()
for header, values := range res.Header {
for _, value := range values {
w.Header().Add(header, value)
}
}
w.WriteHeader(res.StatusCode)
if _, err := io.Copy(w, res.Body); err != nil {
panic(errors.WithStack(err))
}
}

View File

@ -31,6 +31,8 @@ type Handler struct {
server *app.Server
serverModuleFactories []app.ServerModuleFactory
httpClient *http.Client
mutex sync.RWMutex
}
@ -91,6 +93,7 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
sockjsOpts: opts.SockJS,
router: router,
serverModuleFactories: opts.ServerModuleFactories,
httpClient: opts.HTTPClient,
bus: opts.Bus,
}
@ -103,6 +106,8 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
r.Route("/api/v1", func(r chi.Router) {
r.Post("/upload", handler.handleAppUpload)
r.Get("/download/{bucket}/{blobID}", handler.handleAppDownload)
r.Get("/fetch", handler.handleAppFetch)
})
r.HandleFunc("/sock/*", handler.handleSockJS)

View File

@ -1,6 +1,7 @@
package http
import (
"net/http"
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
@ -14,6 +15,7 @@ type HandlerOptions struct {
SockJS sockjs.Options
ServerModuleFactories []app.ServerModuleFactory
UploadMaxFileSize int64
HTTPClient *http.Client
}
func defaultHandlerOptions() *HandlerOptions {
@ -27,6 +29,9 @@ func defaultHandlerOptions() *HandlerOptions {
SockJS: sockjsOptions,
ServerModuleFactories: make([]app.ServerModuleFactory, 0),
UploadMaxFileSize: 10 << (10 * 2), // 10Mb
HTTPClient: &http.Client{
Timeout: time.Second * 30,
},
}
}
@ -55,3 +60,9 @@ func WithUploadMaxFileSize(size int64) HandlerOptionFunc {
opts.UploadMaxFileSize = size
}
}
func WithHTTPClient(client *http.Client) HandlerOptionFunc {
return func(opts *HandlerOptions) {
opts.HTTPClient = client
}
}

View File

@ -0,0 +1,49 @@
package fetch
import (
"context"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/bus"
"github.com/oklog/ulid/v2"
)
const (
MessageNamespaceFetchRequest bus.MessageNamespace = "fetchRequest"
MessageNamespaceFetchResponse bus.MessageNamespace = "fetchResponse"
)
type MessageFetchRequest struct {
Context context.Context
RequestID string
URL *url.URL
RemoteAddr string
}
func (m *MessageFetchRequest) MessageNamespace() bus.MessageNamespace {
return MessageNamespaceFetchRequest
}
func NewMessageFetchRequest(ctx context.Context, remoteAddr string, url *url.URL) *MessageFetchRequest {
return &MessageFetchRequest{
Context: ctx,
RequestID: ulid.Make().String(),
RemoteAddr: remoteAddr,
URL: url,
}
}
type MessageFetchResponse struct {
RequestID string
Allow bool
}
func (m *MessageFetchResponse) MessageNamespace() bus.MessageNamespace {
return MessageNamespaceFetchResponse
}
func NewMessageFetchResponse(requestID string) *MessageFetchResponse {
return &MessageFetchResponse{
RequestID: requestID,
}
}

122
pkg/module/fetch/module.go Normal file
View File

@ -0,0 +1,122 @@
package fetch
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus"
"github.com/dop251/goja"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type Module struct {
server *app.Server
bus bus.Bus
}
func (m *Module) Name() string {
return "fetch"
}
func (m *Module) Export(export *goja.Object) {
funcs := map[string]any{
"get": m.get,
}
for name, fn := range funcs {
if err := export.Set(name, fn); err != nil {
panic(errors.Wrapf(err, "could not set '%s' function", name))
}
}
}
func (m *Module) get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
// ctx := util.AssertContext(call.Argument(0), rt)
return nil
}
func (m *Module) handleMessages() {
ctx := context.Background()
err := m.bus.Reply(ctx, MessageNamespaceFetchRequest, func(msg bus.Message) (bus.Message, error) {
fetchRequest, ok := msg.(*MessageFetchRequest)
if !ok {
return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message fetch request, got '%T'", msg)
}
res, err := m.handleFetchRequest(fetchRequest)
if err != nil {
logger.Error(ctx, "could not handle fetch request", logger.E(errors.WithStack(err)))
return nil, errors.WithStack(err)
}
logger.Debug(ctx, "fetch request response", logger.F("response", res))
return res, nil
})
if err != nil {
panic(errors.WithStack(err))
}
}
func (m *Module) handleFetchRequest(req *MessageFetchRequest) (*MessageFetchResponse, error) {
res := NewMessageFetchResponse(req.RequestID)
ctx := logger.With(
req.Context,
logger.F("url", req.URL.String()),
logger.F("remoteAddr", req.RemoteAddr),
logger.F("requestID", req.RequestID),
)
rawResult, err := m.server.ExecFuncByName(ctx, "onClientFetch", ctx, req.URL.String(), req.RemoteAddr)
if err != nil {
if errors.Is(err, app.ErrFuncDoesNotExist) {
res.Allow = false
return res, nil
}
return nil, errors.WithStack(err)
}
result, ok := rawResult.Export().(map[string]interface{})
if !ok {
return nil, errors.Errorf(
"unexpected onClientFetch result: expected 'map[string]interface{}', got '%T'",
rawResult.Export(),
)
}
var allow bool
rawAllow, exists := result["allow"]
if !exists {
allow = false
} else {
allow, ok = rawAllow.(bool)
if !ok {
return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
}
}
res.Allow = allow
return res, nil
}
func ModuleFactory(bus bus.Bus) app.ServerModuleFactory {
return func(server *app.Server) app.ServerModule {
mod := &Module{
bus: bus,
server: server,
}
go mod.handleMessages()
return mod
}
}

View File

@ -0,0 +1,78 @@
package fetch
import (
"context"
"io/ioutil"
"net/url"
"testing"
"cdr.dev/slog"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/bus/memory"
"forge.cadoles.com/arcad/edge/pkg/module"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestFetchModule(t *testing.T) {
t.Parallel()
logger.SetLevel(slog.LevelDebug)
bus := memory.NewBus()
server := app.NewServer(
module.ContextModuleFactory(),
module.ConsoleModuleFactory(),
ModuleFactory(bus),
)
data, err := ioutil.ReadFile("testdata/fetch.js")
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
if err := server.Load("testdata/fetch.js", string(data)); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer server.Stop()
if err := server.Start(); err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
ctx := context.Background()
remoteAddr := "127.0.0.1"
url, _ := url.Parse("http://example.com")
rawReply, err := bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
reply, ok := rawReply.(*MessageFetchResponse)
if !ok {
t.Fatalf("unexpected reply type '%T'", rawReply)
}
if e, g := true, reply.Allow; e != g {
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
}
url, _ = url.Parse("https://google.com")
rawReply, err = bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
reply, ok = rawReply.(*MessageFetchResponse)
if !ok {
t.Fatalf("unexpected reply type '%T'", rawReply)
}
if e, g := false, reply.Allow; e != g {
t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
}
}

7
pkg/module/fetch/testdata/fetch.js vendored Normal file
View File

@ -0,0 +1,7 @@
var ctx = context.new();
function onClientFetch(ctx, url, remoteAddr) {
if (url === 'http://example.com') return { allow: true };
return { allow: false };
}

View File

@ -4084,6 +4084,9 @@ var Edge = (() => {
blobUrl(bucket, blobId) {
return `/edge/api/v1/download/${bucket}/${blobId}`;
}
externalUrl(url) {
return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`;
}
};
// pkg/sdk/client/src/index.ts

File diff suppressed because one or more lines are too long

View File

@ -264,7 +264,11 @@ export class Client extends EventTarget {
});
}
blobUrl(bucket: string, blobId: string) {
blobUrl(bucket: string, blobId: string): string {
return `/edge/api/v1/download/${bucket}/${blobId}`;
}
externalUrl(url: string): string {
return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`
}
}