fix(storage,document,sqlite): fix nil pointer exception in query options

ref #2

pkg/app/server.go

@@ -1,15 +1,20 @@
 package app
 
 import (
+	"context"
 	"math/rand"
 	"sync"
 
 	"github.com/dop251/goja"
 	"github.com/dop251/goja_nodejs/eventloop"
 	"github.com/pkg/errors"
+	"gitlab.com/wpetit/goweb/logger"
 )
 
-var ErrFuncDoesNotExist = errors.New("function does not exist")
+var (
+	ErrFuncDoesNotExist = errors.New("function does not exist")
+	ErUnknownError      = errors.New("unknown error")
+)
 
 type Server struct {
 	runtime *goja.Runtime
@@ -26,16 +31,18 @@ func (s *Server) Load(name string, src string) error {
 	return nil
 }
 
-func (s *Server) ExecFuncByName(funcName string, args ...interface{}) (goja.Value, error) {
+func (s *Server) ExecFuncByName(ctx context.Context, funcName string, args ...interface{}) (goja.Value, error) {
+	ctx = logger.With(ctx, logger.F("function", funcName), logger.F("args", args))
+
 	callable, ok := goja.AssertFunction(s.runtime.Get(funcName))
 	if !ok {
 		return nil, errors.WithStack(ErrFuncDoesNotExist)
 	}
 
-	return s.Exec(callable, args...)
+	return s.Exec(ctx, callable, args...)
 }
 
-func (s *Server) Exec(callable goja.Callable, args ...interface{}) (goja.Value, error) {
+func (s *Server) Exec(ctx context.Context, callable goja.Callable, args ...interface{}) (goja.Value, error) {
 	var (
 		wg    sync.WaitGroup
 		value goja.Value
@@ -45,6 +52,25 @@ func (s *Server) Exec(callable goja.Callable, args ...interface{}) (goja.Value,
 	wg.Add(1)
 
 	s.loop.RunOnLoop(func(vm *goja.Runtime) {
+		logger.Debug(ctx, "executing callable")
+
+		defer wg.Done()
+
+		defer func() {
+			if recovered := recover(); recovered != nil {
+				revoveredErr, ok := recovered.(error)
+				if ok {
+					logger.Error(ctx, "recovered runtime error", logger.E(errors.WithStack(revoveredErr)))
+
+					err = errors.WithStack(ErUnknownError)
+
+					return
+				}
+
+				panic(recovered)
+			}
+		}()
+
 		jsArgs := make([]goja.Value, 0, len(args))
 		for _, a := range args {
 			jsArgs = append(jsArgs, vm.ToValue(a))
@@ -54,8 +80,6 @@ func (s *Server) Exec(callable goja.Callable, args ...interface{}) (goja.Value,
 		if err != nil {
 			err = errors.WithStack(err)
 		}
-
-		wg.Done()
 	})
 
 	wg.Wait()

pkg/module/auth/module_test.go

@@ -65,7 +65,7 @@ func TestAuthModule(t *testing.T) {
 
 	ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
 
-	if _, err := server.ExecFuncByName("testAuth", ctx); err != nil {
+	if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
 		t.Fatalf("%+v", errors.WithStack(err))
 	}
 }
@@ -104,7 +104,7 @@ func TestAuthAnonymousModule(t *testing.T) {
 
 	ctx := context.WithValue(context.Background(), edgeHTTP.ContextKeyOriginRequest, req)
 
-	if _, err := server.ExecFuncByName("testAuth", ctx); err != nil {
+	if _, err := server.ExecFuncByName(ctx, "testAuth", ctx); err != nil {
 		t.Fatalf("%+v", errors.WithStack(err))
 	}
 }

pkg/module/blob.go

@@ -88,7 +88,7 @@ func (m *BlobModule) handleUploadRequest(req *MessageUploadRequest) (*MessageUpl
 		"contentType": req.FileHeader.Header.Get("Content-Type"),
 	}
 
-	rawResult, err := m.server.ExecFuncByName("onBlobUpload", ctx, blobID, blobInfo, req.Metadata)
+	rawResult, err := m.server.ExecFuncByName(ctx, "onBlobUpload", ctx, blobID, blobInfo, req.Metadata)
 	if err != nil {
 		if errors.Is(err, app.ErrFuncDoesNotExist) {
 			res.Allow = false
@@ -193,7 +193,7 @@ func (m *BlobModule) saveBlob(ctx context.Context, bucketName string, blobID sto
 func (m *BlobModule) handleDownloadRequest(req *MessageDownloadRequest) (*MessageDownloadResponse, error) {
 	res := NewMessageDownloadResponse(req.RequestID)
 
-	rawResult, err := m.server.ExecFuncByName("onBlobDownload", req.Context, req.Bucket, req.BlobID)
+	rawResult, err := m.server.ExecFuncByName(req.Context, "onBlobDownload", req.Context, req.Bucket, req.BlobID)
 	if err != nil {
 		if errors.Is(err, app.ErrFuncDoesNotExist) {
 			res.Allow = false

pkg/module/cast/module_test.go

@@ -1,6 +1,7 @@
 package cast
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -79,7 +80,7 @@ func TestCastModuleRefreshDevices(t *testing.T) {
 
 	defer server.Stop()
 
-	result, err := server.ExecFuncByName("refreshDevices")
+	result, err := server.ExecFuncByName(context.Background(), "refreshDevices")
 	if err != nil {
 		t.Error(errors.WithStack(err))
 	}

pkg/module/lifecycle.go

@@ -21,7 +21,7 @@ func (m *LifecycleModule) Export(export *goja.Object) {
 }
 
 func (m *LifecycleModule) OnInit() error {
-	if _, err := m.server.ExecFuncByName("onInit"); err != nil {
+	if _, err := m.server.ExecFuncByName(context.Background(), "onInit"); err != nil {
 		if errors.Is(err, app.ErrFuncDoesNotExist) {
 			logger.Warn(context.Background(), "could not find onInit() function", logger.E(errors.WithStack(err)))
 

pkg/module/net/module.go

@@ -129,7 +129,7 @@ func (m *Module) handleClientMessages() {
 				logger.F("message", clientMessage),
 			)
 
-			if _, err := m.server.ExecFuncByName("onClientMessage", clientMessage.Context, clientMessage.Data); err != nil {
+			if _, err := m.server.ExecFuncByName(clientMessage.Context, "onClientMessage", clientMessage.Context, clientMessage.Data); err != nil {
 				if errors.Is(err, app.ErrFuncDoesNotExist) {
 					continue
 				}

pkg/module/rpc.go

@@ -161,7 +161,7 @@ func (m *RPCModule) handleMessages() {
 			continue
 		}
 
-		result, err := m.server.Exec(callable, clientMessage.Context, req.Params)
+		result, err := m.server.Exec(clientMessage.Context, callable, clientMessage.Context, req.Params)
 		if err != nil {
 			logger.Error(
 				ctx, "rpc call error",

pkg/module/store/module.go

@@ -102,7 +102,7 @@ func (m *Module) query(call goja.FunctionCall, rt *goja.Runtime) goja.Value {
 		}
 
 		if queryOptions.Offset != nil {
-			queryOptionsFuncs = append(queryOptionsFuncs, storage.WithOffset(*queryOptions.Limit))
+			queryOptionsFuncs = append(queryOptionsFuncs, storage.WithOffset(*queryOptions.Offset))
 		}
 
 		if queryOptions.OrderDirection != nil {

pkg/module/store/module_test.go

@@ -1,6 +1,7 @@
 package store
 
 import (
+	"context"
 	"io/ioutil"
 	"testing"
 
@@ -34,7 +35,7 @@ func TestStoreModule(t *testing.T) {
 		t.Fatalf("%+v", errors.WithStack(err))
 	}
 
-	if _, err := server.ExecFuncByName("testStore"); err != nil {
+	if _, err := server.ExecFuncByName(context.Background(), "testStore"); err != nil {
 		t.Fatalf("%+v", errors.WithStack(err))
 	}
 

pkg/storage/query_options.go

@@ -7,35 +7,35 @@ const (
 	OrderDirectionDesc OrderDirection = "DESC"
 )
 
-type QueryOption struct {
+type QueryOptions struct {
 	Limit          *int
 	Offset         *int
 	OrderBy        *string
 	OrderDirection *OrderDirection
 }
 
-type QueryOptionFunc func(o *QueryOption)
+type QueryOptionFunc func(o *QueryOptions)
 
 func WithLimit(limit int) QueryOptionFunc {
-	return func(o *QueryOption) {
+	return func(o *QueryOptions) {
 		o.Limit = &limit
 	}
 }
 
 func WithOffset(offset int) QueryOptionFunc {
-	return func(o *QueryOption) {
+	return func(o *QueryOptions) {
 		o.Offset = &offset
 	}
 }
 
 func WithOrderBy(orderBy string) QueryOptionFunc {
-	return func(o *QueryOption) {
+	return func(o *QueryOptions) {
 		o.OrderBy = &orderBy
 	}
 }
 
 func WithOrderDirection(direction OrderDirection) QueryOptionFunc {
-	return func(o *QueryOption) {
+	return func(o *QueryOptions) {
 		o.OrderDirection = &direction
 	}
 }

pkg/storage/sqlite/document_store.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"math"
 	"sync"
 	"time"
 
@@ -90,6 +91,11 @@ func (s *DocumentStore) Get(ctx context.Context, collection string, id storage.D
 
 // Query implements storage.DocumentStore
 func (s *DocumentStore) Query(ctx context.Context, collection string, filter *filter.Filter, funcs ...storage.QueryOptionFunc) ([]storage.Document, error) {
+	opts := &storage.QueryOptions{}
+	for _, fn := range funcs {
+		fn(opts)
+	}
+
 	var documents []storage.Document
 
 	err := s.withTx(ctx, func(tx *sql.Tx) error {
@@ -120,6 +126,29 @@ func (s *DocumentStore) Query(ctx context.Context, collection string, filter *fi
 
 		args = append([]interface{}{collection}, args...)
 
+		if opts.OrderBy != nil {
+			direction := storage.OrderDirectionAsc
+			if opts.OrderDirection != nil {
+				direction = *opts.OrderDirection
+			}
+
+			query, args = withOrderByClause(query, args, *opts.OrderBy, direction)
+		}
+
+		if opts.Offset != nil || opts.Limit != nil {
+			offset := 0
+			if opts.Offset != nil {
+				offset = *opts.Offset
+			}
+
+			limit := math.MaxInt
+			if opts.Limit != nil {
+				limit = *opts.Limit
+			}
+
+			query, args = withLimitOffsetClause(query, args, limit, offset)
+		}
+
 		logger.Debug(
 			ctx, "executing query",
 			logger.F("query", query),
@@ -331,6 +360,41 @@ func (s *DocumentStore) ensureTables(ctx context.Context, db *sql.DB) error {
 	return nil
 }
 
+func withOrderByClause(query string, args []any, orderBy string, orderDirection storage.OrderDirection) (string, []any) {
+	direction := "ASC"
+	if orderDirection == storage.OrderDirectionDesc {
+		direction = "DESC"
+	}
+
+	var column string
+
+	switch orderBy {
+	case storage.DocumentAttrID:
+		column = "id"
+
+	case storage.DocumentAttrCreatedAt:
+		column = "created_at"
+
+	case storage.DocumentAttrUpdatedAt:
+		column = "updated_at"
+
+	default:
+		column = fmt.Sprintf("json_extract(data, '$.' || $%d)", len(args)+1)
+		args = append(args, orderBy)
+	}
+
+	query += fmt.Sprintf(` ORDER BY %s %s`, column, direction)
+
+	return query, args
+}
+
+func withLimitOffsetClause(query string, args []any, limit int, offset int) (string, []any) {
+	query += fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
+	args = append(args, limit, offset)
+
+	return query, args
+}
+
 func NewDocumentStore(path string) *DocumentStore {
 	return &DocumentStore{
 		db:       nil,

pkg/storage/testsuite/document_store_ops.go

@@ -41,7 +41,7 @@ var documentStoreOpsTestCases = []documentStoreOpsTestCase{
 				}),
 			)
 
-			results, err := store.Query(ctx, collection, filter, nil)
+			results, err := store.Query(ctx, collection, filter)
 			if err != nil {
 				return errors.WithStack(err)
 			}
@@ -82,7 +82,7 @@ var documentStoreOpsTestCases = []documentStoreOpsTestCase{
 				}),
 			)
 
-			results, err := store.Query(ctx, collection, filter, nil)
+			results, err := store.Query(ctx, collection, filter)
 			if err != nil {
 				return errors.WithStack(err)
 			}
@@ -127,7 +127,7 @@ var documentStoreOpsTestCases = []documentStoreOpsTestCase{
 				),
 			)
 
-			results, err := store.Query(ctx, collection, filter, nil)
+			results, err := store.Query(ctx, collection, filter)
 			if err != nil {
 				return errors.WithStack(err)
 			}
@@ -219,7 +219,7 @@ var documentStoreOpsTestCases = []documentStoreOpsTestCase{
 
 			// Verify that there is no additional created document in the collection
 
-			results, err := store.Query(ctx, collection, nil, nil)
+			results, err := store.Query(ctx, collection, nil)
 			if err != nil {
 				return errors.WithStack(err)
 			}
@@ -228,6 +228,206 @@ var documentStoreOpsTestCases = []documentStoreOpsTestCase{
 				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
 			}
 
+			return nil
+		},
+	},
+	{
+		Name: "Query order by document field",
+		Run: func(ctx context.Context, store storage.DocumentStore) error {
+			docs := []storage.Document{
+				{
+					"sortedField": 0,
+					"name":        "Item 1",
+				},
+				{
+					"sortedField": 1,
+					"name":        "Item 2",
+				},
+				{
+					"sortedField": 2,
+					"name":        "Item 3",
+				},
+			}
+
+			collection := "ordered_query_by_document_field"
+
+			for _, doc := range docs {
+				if _, err := store.Upsert(ctx, collection, doc); err != nil {
+					return errors.WithStack(err)
+				}
+			}
+
+			results, err := store.Query(
+				ctx, collection, nil,
+				storage.WithOrderBy("sortedField"),
+				storage.WithOrderDirection(storage.OrderDirectionAsc),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 3, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[0]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[2]["name"], results[2]["name"]; e != g {
+				return errors.Errorf("results[2][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			results, err = store.Query(
+				ctx, collection, nil,
+				storage.WithOrderBy("sortedField"),
+				storage.WithOrderDirection(storage.OrderDirectionDesc),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 3, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[2]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[0]["name"], results[2]["name"]; e != g {
+				return errors.Errorf("results[2][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			return nil
+		},
+	},
+	{
+		Name: "Query order by special attr",
+		Run: func(ctx context.Context, store storage.DocumentStore) error {
+			docs := []storage.Document{
+				{
+					"name": "Item 1",
+				},
+				{
+					"name": "Item 2",
+				},
+				{
+					"name": "Item 3",
+				},
+			}
+
+			collection := "ordered_query_by_special_attr"
+
+			for _, doc := range docs {
+				if _, err := store.Upsert(ctx, collection, doc); err != nil {
+					return errors.WithStack(err)
+				}
+			}
+
+			results, err := store.Query(
+				ctx, collection, nil,
+				storage.WithOrderBy(storage.DocumentAttrCreatedAt),
+				storage.WithOrderDirection(storage.OrderDirectionAsc),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 3, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[0]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[2]["name"], results[2]["name"]; e != g {
+				return errors.Errorf("results[2][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			results, err = store.Query(
+				ctx, collection, nil,
+				storage.WithOrderBy(storage.DocumentAttrCreatedAt),
+				storage.WithOrderDirection(storage.OrderDirectionDesc),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 3, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[2]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[0]["name"], results[2]["name"]; e != g {
+				return errors.Errorf("results[2][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			return nil
+		},
+	},
+	{
+		Name: "Query limit and offset",
+		Run: func(ctx context.Context, store storage.DocumentStore) error {
+			docs := []storage.Document{
+				{"name": "Item 1"},
+				{"name": "Item 2"},
+				{"name": "Item 3"},
+				{"name": "Item 4"},
+			}
+
+			collection := "query_limit_and_offset"
+
+			for _, doc := range docs {
+				if _, err := store.Upsert(ctx, collection, doc); err != nil {
+					return errors.WithStack(err)
+				}
+			}
+
+			results, err := store.Query(
+				ctx, collection, nil,
+				storage.WithLimit(2),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 2, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[0]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[1]["name"], results[1]["name"]; e != g {
+				return errors.Errorf("results[1][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			results, err = store.Query(
+				ctx, collection, nil,
+				storage.WithOffset(2),
+			)
+			if err != nil {
+				return errors.WithStack(err)
+			}
+
+			if e, g := 2, len(results); e != g {
+				return errors.Errorf("len(results): expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[2]["name"], results[0]["name"]; e != g {
+				return errors.Errorf("results[0][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
+			if e, g := docs[3]["name"], results[1]["name"]; e != g {
+				return errors.Errorf("results[1][\"name\"]: expected '%v', got '%v'", e, g)
+			}
+
 			return nil
 		},
 	},