feat(app,module): fetch basic module

cmd/cli/command/app/run.go

@@ -21,6 +21,7 @@ import (
 	authHTTP "forge.cadoles.com/arcad/edge/pkg/module/auth/http"
 	"forge.cadoles.com/arcad/edge/pkg/module/blob"
 	"forge.cadoles.com/arcad/edge/pkg/module/cast"
+	"forge.cadoles.com/arcad/edge/pkg/module/fetch"
 	"forge.cadoles.com/arcad/edge/pkg/module/net"
 	"forge.cadoles.com/arcad/edge/pkg/storage"
 	"forge.cadoles.com/arcad/edge/pkg/storage/sqlite"
@@ -208,6 +209,7 @@ func getServerModules(bus bus.Bus, ds storage.DocumentStore, bs storage.BlobStor
 			},
 			manifest,
 		)),
+		fetch.ModuleFactory(bus),
 	}
 }
 

doc/apps/client-api/README.md

@@ -51,6 +51,10 @@ Edge.connect().then(() => {
 
 > `TODO`
 
+### `Edge.externalUrl(url: string): string`
+
+Retourne une URL "locale" permettant d'accéder à une ressource externe, en fonction de règles propres à l'application. Voir module [`fetch`](../server-api/fetch.md).
+
 ## Événements
 
 ### `"message"`

doc/apps/server-api/README.md

@@ -26,6 +26,7 @@ Listes des modules disponibles côté serveur.
 - [`cast`](./cast.md)
 - [`console`](./console.md)
 - [`context`](./context.md)
+- [`fetch`](./fetch.md)
 - [`net`](./net.md)
 - [`rpc`](./rpc.md)
 - [`store`](./store.md)

doc/apps/server-api/fetch.md

@@ -0,0 +1,33 @@
+# Module `fetch`
+
+Ce module permet l'accès à des ressources distantes (sur Internet) depuis votre application.
+
+## Fonctions de rappel
+
+Pour permettre aux utilisateurs d'accéder à des ressources distantes, vous devez déclarer la fonction `onClientFetch(ctx: Context, url: string, remoteAddr: string)` dans le fichier `server/main.js` de votre application.
+
+### `onClientFetch(ctx: Context, url: string, remoteAddr: string)`
+
+#### Usage
+
+**Côté client**
+```js
+// Création d'une URL "locale" permettant d'accéder à la ressource distante
+var url = Edge.externalUrl("http://example.com")
+
+// Vous pouvez utiliser l'URL comme attribut `src` d'une balise <img> par exemple
+// ou effectuer une requête fetch() avec celle ci.
+fetch(url).then(res => res.text()).then(content => console.log(content));
+```
+
+**Côté serveur**
+```js
+function onClientFetch(ctx, url, remoteAddr) {
+    // Autoriser la récupération de l'URL demandée ou non
+    // Dans cet exemple, seule l'URL externe 'http://example.com' est autorisée
+    // Les autres URLs recevront une erreur HTTP 403 - Forbidden
+    var authorized = url === "http://example.com"
+
+    return { allow: authorized };
+}
+```

misc/client-sdk-testsuite/src/public/index.html

@@ -26,6 +26,7 @@
     <script src="/test/rpc-module.js"></script>
     <script src="/test/file-module.js"></script>
     <script src="/test/app-module.js"></script>
+    <script src="/test/fetch-module.js"></script>
     <script class="mocha-exec">
       mocha.run();
     </script>

misc/client-sdk-testsuite/src/public/test/fetch-module.js

@@ -0,0 +1,33 @@
+describe('Fetch Module', function () {
+
+    before(() => {
+        return Edge.connect();
+    });
+
+    after(() => {
+        Edge.disconnect();
+    });
+
+    it('should fetch an authorized external url', function () {
+        var externalUrl = Edge.externalUrl("http://example.com");
+
+        return fetch(externalUrl)
+            .then(res => {
+                chai.assert.equal(res.status, 200)
+                return res.text()
+            })
+            .then(content => {
+                chai.assert.include(content, '<h1>Example Domain</h1>')
+            })
+    });
+
+    it('should not fetch an unauthorized external url', function () {
+        var externalUrl = Edge.externalUrl("https://google.com");
+
+        return fetch(externalUrl)
+            .then(res => {
+                chai.assert.equal(res.status, 403)
+            })
+    });
+
+});

misc/client-sdk-testsuite/src/server/main.js

@@ -98,3 +98,7 @@ function getAppUrl(ctx, params) {
   var appId = params.appId;
   return app.getUrl(ctx, appId);
 }
+
+function onClientFetch(ctx, url, remoteAddr) {
+  return { allow: url === 'http://example.com' };
+}

pkg/http/fetch.go

@@ -0,0 +1,112 @@
+package http
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+
+	"forge.cadoles.com/arcad/edge/pkg/module"
+	"forge.cadoles.com/arcad/edge/pkg/module/fetch"
+	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
+)
+
+func (h *Handler) handleAppFetch(w http.ResponseWriter, r *http.Request) {
+	h.mutex.RLock()
+	defer h.mutex.RUnlock()
+
+	ctx := r.Context()
+
+	ctx = module.WithContext(ctx, map[module.ContextKey]any{
+		ContextKeyOriginRequest: r,
+	})
+
+	rawURL := r.URL.Query().Get("url")
+
+	url, err := url.Parse(rawURL)
+	if err != nil {
+		jsonError(w, http.StatusBadRequest, errorCodeBadRequest)
+
+		return
+	}
+
+	requestMsg := fetch.NewMessageFetchRequest(ctx, r.RemoteAddr, url)
+
+	reply, err := h.bus.Request(ctx, requestMsg)
+	if err != nil {
+		logger.Error(ctx, "could not retrieve fetch request reply", logger.E(errors.WithStack(err)))
+		jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
+
+		return
+	}
+
+	logger.Debug(ctx, "fetch reply", logger.F("reply", reply))
+
+	responseMsg, ok := reply.(*fetch.MessageFetchResponse)
+	if !ok {
+		logger.Error(
+			ctx, "unexpected fetch response message",
+			logger.F("message", reply),
+		)
+		jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
+
+		return
+	}
+
+	if !responseMsg.Allow {
+		jsonError(w, http.StatusForbidden, errorCodeForbidden)
+
+		return
+	}
+
+	proxyReq, err := http.NewRequest(http.MethodGet, url.String(), nil)
+	if err != nil {
+		logger.Error(
+			ctx, "could not create proxy request",
+			logger.E(errors.WithStack(err)),
+		)
+		jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
+
+		return
+	}
+
+	for header, values := range r.Header {
+		for _, value := range values {
+			proxyReq.Header.Add(header, value)
+		}
+	}
+
+	proxyReq.Header.Add("X-Forwarded-From", r.RemoteAddr)
+
+	res, err := h.httpClient.Do(proxyReq)
+	if err != nil {
+		logger.Error(
+			ctx, "could not execute proxy request",
+			logger.E(errors.WithStack(err)),
+		)
+		jsonError(w, http.StatusInternalServerError, errorCodeInternalError)
+
+		return
+	}
+
+	defer func() {
+		if err := res.Body.Close(); err != nil {
+			logger.Error(
+				ctx, "could not close response body",
+				logger.E(errors.WithStack(err)),
+			)
+		}
+	}()
+
+	for header, values := range res.Header {
+		for _, value := range values {
+			w.Header().Add(header, value)
+		}
+	}
+
+	w.WriteHeader(res.StatusCode)
+
+	if _, err := io.Copy(w, res.Body); err != nil {
+		panic(errors.WithStack(err))
+	}
+}

pkg/http/handler.go

@@ -31,6 +31,8 @@ type Handler struct {
 	server                *app.Server
 	serverModuleFactories []app.ServerModuleFactory
 
+	httpClient *http.Client
+
 	mutex sync.RWMutex
 }
 
@@ -91,6 +93,7 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
 		sockjsOpts:            opts.SockJS,
 		router:                router,
 		serverModuleFactories: opts.ServerModuleFactories,
+		httpClient:            opts.HTTPClient,
 		bus:                   opts.Bus,
 	}
 
@@ -103,6 +106,8 @@ func NewHandler(funcs ...HandlerOptionFunc) *Handler {
 		r.Route("/api/v1", func(r chi.Router) {
 			r.Post("/upload", handler.handleAppUpload)
 			r.Get("/download/{bucket}/{blobID}", handler.handleAppDownload)
+
+			r.Get("/fetch", handler.handleAppFetch)
 		})
 
 		r.HandleFunc("/sock/*", handler.handleSockJS)

pkg/http/options.go

@@ -1,6 +1,7 @@
 package http
 
 import (
+	"net/http"
 	"time"
 
 	"forge.cadoles.com/arcad/edge/pkg/app"
@@ -14,6 +15,7 @@ type HandlerOptions struct {
 	SockJS                sockjs.Options
 	ServerModuleFactories []app.ServerModuleFactory
 	UploadMaxFileSize     int64
+	HTTPClient            *http.Client
 }
 
 func defaultHandlerOptions() *HandlerOptions {
@@ -27,6 +29,9 @@ func defaultHandlerOptions() *HandlerOptions {
 		SockJS:                sockjsOptions,
 		ServerModuleFactories: make([]app.ServerModuleFactory, 0),
 		UploadMaxFileSize:     10 << (10 * 2), // 10Mb
+		HTTPClient: &http.Client{
+			Timeout: time.Second * 30,
+		},
 	}
 }
 
@@ -55,3 +60,9 @@ func WithUploadMaxFileSize(size int64) HandlerOptionFunc {
 		opts.UploadMaxFileSize = size
 	}
 }
+
+func WithHTTPClient(client *http.Client) HandlerOptionFunc {
+	return func(opts *HandlerOptions) {
+		opts.HTTPClient = client
+	}
+}

pkg/module/fetch/fetch_message.go

@@ -0,0 +1,49 @@
+package fetch
+
+import (
+	"context"
+	"net/url"
+
+	"forge.cadoles.com/arcad/edge/pkg/bus"
+	"github.com/oklog/ulid/v2"
+)
+
+const (
+	MessageNamespaceFetchRequest  bus.MessageNamespace = "fetchRequest"
+	MessageNamespaceFetchResponse bus.MessageNamespace = "fetchResponse"
+)
+
+type MessageFetchRequest struct {
+	Context    context.Context
+	RequestID  string
+	URL        *url.URL
+	RemoteAddr string
+}
+
+func (m *MessageFetchRequest) MessageNamespace() bus.MessageNamespace {
+	return MessageNamespaceFetchRequest
+}
+
+func NewMessageFetchRequest(ctx context.Context, remoteAddr string, url *url.URL) *MessageFetchRequest {
+	return &MessageFetchRequest{
+		Context:    ctx,
+		RequestID:  ulid.Make().String(),
+		RemoteAddr: remoteAddr,
+		URL:        url,
+	}
+}
+
+type MessageFetchResponse struct {
+	RequestID string
+	Allow     bool
+}
+
+func (m *MessageFetchResponse) MessageNamespace() bus.MessageNamespace {
+	return MessageNamespaceFetchResponse
+}
+
+func NewMessageFetchResponse(requestID string) *MessageFetchResponse {
+	return &MessageFetchResponse{
+		RequestID: requestID,
+	}
+}

pkg/module/fetch/module.go

@@ -0,0 +1,122 @@
+package fetch
+
+import (
+	"context"
+
+	"forge.cadoles.com/arcad/edge/pkg/app"
+	"forge.cadoles.com/arcad/edge/pkg/bus"
+	"github.com/dop251/goja"
+	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
+)
+
+type Module struct {
+	server *app.Server
+	bus    bus.Bus
+}
+
+func (m *Module) Name() string {
+	return "fetch"
+}
+
+func (m *Module) Export(export *goja.Object) {
+	funcs := map[string]any{
+		"get": m.get,
+	}
+
+	for name, fn := range funcs {
+		if err := export.Set(name, fn); err != nil {
+			panic(errors.Wrapf(err, "could not set '%s' function", name))
+		}
+	}
+}
+
+func (m *Module) get(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
+	// ctx := util.AssertContext(call.Argument(0), rt)
+
+	return nil
+}
+
+func (m *Module) handleMessages() {
+	ctx := context.Background()
+
+	err := m.bus.Reply(ctx, MessageNamespaceFetchRequest, func(msg bus.Message) (bus.Message, error) {
+		fetchRequest, ok := msg.(*MessageFetchRequest)
+		if !ok {
+			return nil, errors.Wrapf(bus.ErrUnexpectedMessage, "expected message fetch request, got '%T'", msg)
+		}
+
+		res, err := m.handleFetchRequest(fetchRequest)
+		if err != nil {
+			logger.Error(ctx, "could not handle fetch request", logger.E(errors.WithStack(err)))
+
+			return nil, errors.WithStack(err)
+		}
+
+		logger.Debug(ctx, "fetch request response", logger.F("response", res))
+
+		return res, nil
+	})
+	if err != nil {
+		panic(errors.WithStack(err))
+	}
+}
+
+func (m *Module) handleFetchRequest(req *MessageFetchRequest) (*MessageFetchResponse, error) {
+	res := NewMessageFetchResponse(req.RequestID)
+
+	ctx := logger.With(
+		req.Context,
+		logger.F("url", req.URL.String()),
+		logger.F("remoteAddr", req.RemoteAddr),
+		logger.F("requestID", req.RequestID),
+	)
+
+	rawResult, err := m.server.ExecFuncByName(ctx, "onClientFetch", ctx, req.URL.String(), req.RemoteAddr)
+	if err != nil {
+		if errors.Is(err, app.ErrFuncDoesNotExist) {
+			res.Allow = false
+
+			return res, nil
+		}
+
+		return nil, errors.WithStack(err)
+	}
+
+	result, ok := rawResult.Export().(map[string]interface{})
+	if !ok {
+		return nil, errors.Errorf(
+			"unexpected onClientFetch result: expected 'map[string]interface{}', got '%T'",
+			rawResult.Export(),
+		)
+	}
+
+	var allow bool
+
+	rawAllow, exists := result["allow"]
+	if !exists {
+		allow = false
+	} else {
+		allow, ok = rawAllow.(bool)
+		if !ok {
+			return nil, errors.Errorf("invalid 'allow' result property: got type '%T', expected type '%T'", rawAllow, false)
+		}
+	}
+
+	res.Allow = allow
+
+	return res, nil
+}
+
+func ModuleFactory(bus bus.Bus) app.ServerModuleFactory {
+	return func(server *app.Server) app.ServerModule {
+		mod := &Module{
+			bus:    bus,
+			server: server,
+		}
+
+		go mod.handleMessages()
+
+		return mod
+	}
+}

pkg/module/fetch/module_test.go

@@ -0,0 +1,78 @@
+package fetch
+
+import (
+	"context"
+	"io/ioutil"
+	"net/url"
+	"testing"
+
+	"cdr.dev/slog"
+	"forge.cadoles.com/arcad/edge/pkg/app"
+	"forge.cadoles.com/arcad/edge/pkg/bus/memory"
+	"forge.cadoles.com/arcad/edge/pkg/module"
+	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
+)
+
+func TestFetchModule(t *testing.T) {
+	t.Parallel()
+
+	logger.SetLevel(slog.LevelDebug)
+
+	bus := memory.NewBus()
+
+	server := app.NewServer(
+		module.ContextModuleFactory(),
+		module.ConsoleModuleFactory(),
+		ModuleFactory(bus),
+	)
+
+	data, err := ioutil.ReadFile("testdata/fetch.js")
+	if err != nil {
+		t.Fatalf("%+v", errors.WithStack(err))
+	}
+
+	if err := server.Load("testdata/fetch.js", string(data)); err != nil {
+		t.Fatalf("%+v", errors.WithStack(err))
+	}
+
+	defer server.Stop()
+
+	if err := server.Start(); err != nil {
+		t.Fatalf("%+v", errors.WithStack(err))
+	}
+
+	ctx := context.Background()
+	remoteAddr := "127.0.0.1"
+	url, _ := url.Parse("http://example.com")
+
+	rawReply, err := bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
+	if err != nil {
+		t.Fatalf("%+v", errors.WithStack(err))
+	}
+
+	reply, ok := rawReply.(*MessageFetchResponse)
+	if !ok {
+		t.Fatalf("unexpected reply type '%T'", rawReply)
+	}
+
+	if e, g := true, reply.Allow; e != g {
+		t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
+	}
+
+	url, _ = url.Parse("https://google.com")
+
+	rawReply, err = bus.Request(ctx, NewMessageFetchRequest(ctx, remoteAddr, url))
+	if err != nil {
+		t.Fatalf("%+v", errors.WithStack(err))
+	}
+
+	reply, ok = rawReply.(*MessageFetchResponse)
+	if !ok {
+		t.Fatalf("unexpected reply type '%T'", rawReply)
+	}
+
+	if e, g := false, reply.Allow; e != g {
+		t.Errorf("reply.Allow: expected '%v', got '%v'", e, g)
+	}
+}

pkg/module/fetch/testdata/fetch.js

@@ -0,0 +1,7 @@
+
+var ctx = context.new();
+
+function onClientFetch(ctx, url, remoteAddr) {
+    if (url === 'http://example.com') return { allow: true };
+    return { allow: false };
+}

pkg/sdk/client/dist/client.js

@@ -4084,6 +4084,9 @@ var Edge = (() => {
     blobUrl(bucket, blobId) {
       return `/edge/api/v1/download/${bucket}/${blobId}`;
     }
+    externalUrl(url) {
+      return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`;
+    }
   };
 
   // pkg/sdk/client/src/index.ts

pkg/sdk/client/src/client.ts

@@ -264,7 +264,11 @@ export class Client extends EventTarget {
     });
   }
 
-  blobUrl(bucket: string, blobId: string) {
+  blobUrl(bucket: string, blobId: string): string {
     return `/edge/api/v1/download/${bucket}/${blobId}`;
   }
+
+  externalUrl(url: string): string {
+    return `/edge/api/v1/fetch?url=${encodeURIComponent(url)}`
+  }
 }