feat: make query preperation a JIT operation to improve startup time

core/api.go

@@ -49,6 +49,7 @@ import (
 	"crypto/sha256"
 	"database/sql"
 	"encoding/json"
+	"hash/maphash"
 	_log "log"
 	"os"
 
@@ -83,7 +84,8 @@ type SuperGraph struct {
 	schema      *psql.DBSchema
 	allowList   *allow.List
 	encKey      [32]byte
-	prepared    map[string]*preparedItem
+	hashSeed    maphash.Seed
+	queries     map[uint64]*query
 	roles       map[string]*Role
 	getRole     *sql.Stmt
 	rmap        map[uint64]*resolvFn
@@ -111,6 +113,7 @@ func newSuperGraph(conf *Config, db *sql.DB, dbinfo *psql.DBInfo) (*SuperGraph,
 		db:       db,
 		dbinfo:   dbinfo,
 		log:      _log.New(os.Stdout, "", 0),
+		hashSeed: maphash.MakeSeed(),
 	}
 
 	if err := sg.initConfig(); err != nil {

core/bench.11

@@ -0,0 +1,41 @@
+INF roles_query not defined: attribute based access control disabled
+all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
+goos: darwin
+goarch: amd64
+pkg: github.com/dosco/super-graph/core
+BenchmarkGraphQL-16    	INF roles_query not defined: attribute based access control disabled
+all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
+INF roles_query not defined: attribute based access control disabled
+all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
+INF roles_query not defined: attribute based access control disabled
+all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
+  105048	     10398 ns/op	   18342 B/op	      55 allocs/op
+PASS
+ok  	github.com/dosco/super-graph/core	1.328s
+PASS
+ok  	github.com/dosco/super-graph/core/internal/allow	0.088s
+?   	github.com/dosco/super-graph/core/internal/crypto	[no test files]
+?   	github.com/dosco/super-graph/core/internal/integration_tests	[no test files]
+PASS
+ok  	github.com/dosco/super-graph/core/internal/integration_tests/cockroachdb	0.121s
+PASS
+ok  	github.com/dosco/super-graph/core/internal/integration_tests/postgresql	0.118s
+goos: darwin
+goarch: amd64
+pkg: github.com/dosco/super-graph/core/internal/psql
+BenchmarkCompile-16            	   79845	     14428 ns/op	    4584 B/op	      39 allocs/op
+BenchmarkCompileParallel-16    	  326205	      3918 ns/op	    4633 B/op	      39 allocs/op
+PASS
+ok  	github.com/dosco/super-graph/core/internal/psql	2.696s
+goos: darwin
+goarch: amd64
+pkg: github.com/dosco/super-graph/core/internal/qcode
+BenchmarkQCompile-16        	  146953	      8049 ns/op	    3756 B/op	      28 allocs/op
+BenchmarkQCompileP-16       	  475936	      2447 ns/op	    3790 B/op	      28 allocs/op
+BenchmarkParse-16           	  140811	      8163 ns/op	    3902 B/op	      18 allocs/op
+BenchmarkParseP-16          	  571345	      2041 ns/op	    3903 B/op	      18 allocs/op
+BenchmarkSchemaParse-16     	  230715	      5012 ns/op	    3968 B/op	      57 allocs/op
+BenchmarkSchemaParseP-16    	  802426	      1565 ns/op	    3968 B/op	      57 allocs/op
+PASS
+ok  	github.com/dosco/super-graph/core/internal/qcode	8.427s
+?   	github.com/dosco/super-graph/core/internal/util	[no test files]

core/core.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"hash/maphash"
 	"time"
 
 	"github.com/dosco/super-graph/core/internal/psql"
@@ -165,32 +166,43 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
 
 	} else {
 		role = c.role
-
 	}
 
 	c.res.role = role
 
-	ps, ok := c.sg.prepared[stmtHash(c.res.name, role)]
+	h := maphash.Hash{}
+	h.SetSeed(c.sg.hashSeed)
+
+	q, ok := c.sg.queries[queryID(&h, c.res.name, role)]
 	if !ok {
 		return nil, nil, errNotFound
 	}
-	c.res.sql = ps.st.sql
+
+	if q.sd == nil {
+		q.Do(func() { c.sg.prepare(q, role) })
+
+		if q.err != nil {
+			return nil, nil, err
+		}
+	}
+
+	c.res.sql = q.st.sql
 
 	var root []byte
 	var row *sql.Row
 
-	varsList, err := c.argList(ps.st.md)
+	varsList, err := c.argList(q.st.md)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	if useTx {
-		row = tx.Stmt(ps.sd).QueryRow(varsList...)
+		row = tx.Stmt(q.sd).QueryRow(varsList...)
 	} else {
-		row = ps.sd.QueryRow(varsList...)
+		row = q.sd.QueryRow(varsList...)
 	}
 
-	if ps.roleArg {
+	if q.roleArg {
 		err = row.Scan(&role, &root)
 	} else {
 		err = row.Scan(&root)
@@ -204,15 +216,15 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
 
 	if useTx {
 		if err := tx.Commit(); err != nil {
-			return nil, nil, err
+			return nil, nil, q.err
 		}
 	}
 
-	if root, err = c.sg.encryptCursor(ps.st.qc, root); err != nil {
+	if root, err = c.sg.encryptCursor(q.st.qc, root); err != nil {
 		return nil, nil, err
 	}
 
-	return root, &ps.st, nil
+	return root, &q.st, nil
 }
 
 func (c *scontext) resolveSQL() ([]byte, *stmt, error) {

core/prepare.go

@@ -2,120 +2,97 @@ package core
 
 import (
 	"bytes"
-	"context"
-	"crypto/sha256"
 	"database/sql"
-	"encoding/hex"
 	"fmt"
+	"hash/maphash"
 	"io"
 	"strings"
+	"sync"
 
 	"github.com/dosco/super-graph/core/internal/allow"
 	"github.com/dosco/super-graph/core/internal/qcode"
 )
 
-type preparedItem struct {
+type query struct {
+	sync.Once
 	sd      *sql.Stmt
+	ai      allow.Item
+	qt      qcode.QType
+	err     error
 	st      stmt
 	roleArg bool
 }
 
-func (sg *SuperGraph) initPrepared() error {
+func (sg *SuperGraph) prepare(q *query, role string) {
-	ct := context.Background()
+	var stmts []stmt
+	var err error
 
+	qb := []byte(q.ai.Query)
+
+	switch q.qt {
+	case qcode.QTQuery:
+		if sg.abacEnabled {
+			stmts, err = sg.buildMultiStmt(qb, q.ai.Vars)
+		} else {
+			stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role)
+		}
+
+	case qcode.QTMutation:
+		stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role)
+	}
+
+	if err != nil {
+		sg.log.Printf("WRN %s %s: %v", q.qt, q.ai.Name, err)
+	}
+
+	q.st = stmts[0]
+	q.roleArg = len(stmts) > 1
+
+	q.sd, err = sg.db.Prepare(q.st.sql)
+	if err != nil {
+		q.err = fmt.Errorf("prepare failed: %v: %s", err, q.st.sql)
+	}
+}
+
+func (sg *SuperGraph) initPrepared() error {
 	if sg.allowList.IsPersist() {
 		return nil
 	}
-	sg.prepared = make(map[string]*preparedItem)
 
-	tx, err := sg.db.BeginTx(ct, nil)
+	if err := sg.prepareRoleStmt(); err != nil {
-	if err != nil {
+		return fmt.Errorf("role query: %w", err)
-		return err
-	}
-	defer tx.Rollback() //nolint: errcheck
-
-	if err = sg.prepareRoleStmt(tx); err != nil {
-		return fmt.Errorf("prepareRoleStmt: %w", err)
 	}
 
-	if err := tx.Commit(); err != nil {
+	sg.queries = make(map[uint64]*query)
-		return err
-	}
-
-	success := 0
 
 	list, err := sg.allowList.Load()
 	if err != nil {
 		return err
 	}
 
+	h := maphash.Hash{}
+	h.SetSeed(sg.hashSeed)
+
 	for _, v := range list {
 		if len(v.Query) == 0 {
 			continue
 		}
+		q := &query{ai: v, qt: qcode.GetQType(v.Query)}
 
-		err := sg.prepareStmt(v)
+		switch q.qt {
-		if err != nil {
-			return err
-		} else {
-			success++
-		}
-	}
-
-	sg.log.Printf("INF allow list: prepared %d / %d queries", success, len(list))
-
-	return nil
-}
-
-func (sg *SuperGraph) prepareStmt(item allow.Item) error {
-	query := item.Query
-	qb := []byte(query)
-	vars := item.Vars
-
-	qt := qcode.GetQType(query)
-	ct := context.Background()
-
-	switch qt {
 		case qcode.QTQuery:
-		var stmts1 []stmt
+			sg.queries[queryID(&h, v.Name, "user")] = q
-		var err error
+			h.Reset()
-
-		if sg.abacEnabled {
-			stmts1, err = sg.buildMultiStmt(qb, vars)
-		} else {
-			stmts1, err = sg.buildRoleStmt(qb, vars, "user")
-		}
-
-		if err == nil {
-			if err = sg.prepare(ct, stmts1, stmtHash(item.Name, "user")); err != nil {
-				return err
-			}
-		} else {
-			sg.log.Printf("WRN query %s: %v", item.Name, err)
-		}
 
 			if sg.anonExists {
-			stmts2, err := sg.buildRoleStmt(qb, vars, "anon")
+				sg.queries[queryID(&h, v.Name, "anon")] = q
-
+				h.Reset()
-			if err == nil {
-				if err = sg.prepare(ct, stmts2, stmtHash(item.Name, "anon")); err != nil {
-					return err
-				}
-			} else {
-				sg.log.Printf("WRN query %s: %v", item.Name, err)
-			}
 			}
 
 		case qcode.QTMutation:
 			for _, role := range sg.conf.Roles {
-			stmts, err := sg.buildRoleStmt(qb, vars, role.Name)
+				sg.queries[queryID(&h, v.Name, role.Name)] = q
-
+				h.Reset()
-			if err == nil {
-				if err = sg.prepare(ct, stmts, stmtHash(item.Name, role.Name)); err != nil {
-					return err
-				}
-			} else {
-				sg.log.Printf("WRN mutation %s: %v", item.Name, err)
 			}
 		}
 	}
@@ -123,22 +100,8 @@ func (sg *SuperGraph) prepareStmt(item allow.Item) error {
 	return nil
 }
 
-func (sg *SuperGraph) prepare(ct context.Context, st []stmt, key string) error {
-	sd, err := sg.db.PrepareContext(ct, st[0].sql)
-	if err != nil {
-		return fmt.Errorf("prepare failed: %v: %s", err, st[0].sql)
-	}
-
-	sg.prepared[key] = &preparedItem{
-		sd:      sd,
-		st:      st[0],
-		roleArg: len(st) > 1,
-	}
-	return nil
-}
-
 // nolint: errcheck
-func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error {
+func (sg *SuperGraph) prepareRoleStmt() error {
 	var err error
 
 	if !sg.abacEnabled {
@@ -169,7 +132,7 @@ func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error {
 	io.WriteString(w, `) AS "_sg_auth_roles_query" LIMIT 1) `)
 	io.WriteString(w, `ELSE 'anon' END) FROM (VALUES (1)) AS "_sg_auth_filler" LIMIT 1; `)
 
-	sg.getRole, err = tx.Prepare(w.String())
+	sg.getRole, err = sg.db.Prepare(w.String())
 	if err != nil {
 		return err
 	}
@@ -200,9 +163,8 @@ func (sg *SuperGraph) initAllowList() error {
 }
 
 // nolint: errcheck
-func stmtHash(name string, role string) string {
+func queryID(h *maphash.Hash, name string, role string) uint64 {
-	h := sha256.New()
+	h.WriteString(name)
-	io.WriteString(h, strings.ToLower(name))
+	h.WriteString(role)
-	io.WriteString(h, role)
+	return h.Sum64()
-	return hex.EncodeToString(h.Sum(nil))
 }

internal/serv/internal/migrate/migrate.go

@@ -6,9 +6,11 @@ import (
 	"database/sql"
 	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"text/template"
@@ -105,39 +107,40 @@ func (defaultMigratorFS) Glob(pattern string) ([]string, error) {
 func FindMigrationsEx(path string, fs MigratorFS) ([]string, error) {
 	path = strings.TrimRight(path, string(filepath.Separator))
 
-	fileInfos, err := fs.ReadDir(path)
+	files, err := ioutil.ReadDir(path)
 	if err != nil {
-		return nil, err
+		log.Fatal(err)
 	}
 
-	paths := make([]string, 0, len(fileInfos))
+	fm := make(map[int]string, len(files))
-	for _, fi := range fileInfos {
+	keys := make([]int, 0, len(files))
+
+	for _, fi := range files {
 		if fi.IsDir() {
 			continue
 		}
 
 		matches := migrationPattern.FindStringSubmatch(fi.Name())
+
 		if len(matches) != 2 {
 			continue
 		}
 
-		n, err := strconv.ParseInt(matches[1], 10, 32)
+		n, err := strconv.Atoi(matches[1])
 		if err != nil {
 			// The regexp already validated that the prefix is all digits so this *should* never fail
 			return nil, err
 		}
 
-		mcount := len(paths)
+		fm[n] = filepath.Join(path, fi.Name())
-
+		keys = append(keys, n)
-		if n < int64(mcount) {
-			return nil, fmt.Errorf("Duplicate migration %d", n)
 	}
 
-		if int64(mcount) < n {
+	sort.Ints(keys)
-			return nil, fmt.Errorf("Missing migration %d", mcount)
-		}
 
-		paths = append(paths, filepath.Join(path, fi.Name()))
+	paths := make([]string, 0, len(keys))
+	for _, k := range keys {
+		paths = append(paths, fm[k])
 	}
 
 	return paths, nil