wpetit/super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

feat: make query preperation a JIT operation to improve startup time

Browse Source
This commit is contained in:
Vikram Rangnekar 2020-06-03 01:03:12 -04:00
parent 816121fbcf
commit 1a15e433ba
5 changed files with 150 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/api.go
View File

@ -49,6 +49,7 @@ import (
"crypto/sha256"
"database/sql"
"encoding/json"
"hash/maphash"
_log "log"
"os"
@ -83,7 +84,8 @@ type SuperGraph struct {
schema *psql.DBSchema
allowList *allow.List
encKey [32]byte
prepared map[string]*preparedItem
hashSeed maphash.Seed
queries map[uint64]*query
roles map[string]*Role
getRole *sql.Stmt
rmap map[uint64]*resolvFn
@ -107,10 +109,11 @@ func newSuperGraph(conf *Config, db *sql.DB, dbinfo *psql.DBInfo) (*SuperGraph,
}
sg := &SuperGraph{
conf: conf,
db: db,
dbinfo: dbinfo,
log: _log.New(os.Stdout, "", 0),
conf: conf,
db: db,
dbinfo: dbinfo,
log: _log.New(os.Stdout, "", 0),
hashSeed: maphash.MakeSeed(),
}
if err := sg.initConfig(); err != nil {

41
core/bench.11 Normal file
View File

@ -0,0 +1,41 @@
INF roles_query not defined: attribute based access control disabled
all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/core
BenchmarkGraphQL-16 INF roles_query not defined: attribute based access control disabled
all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
INF roles_query not defined: attribute based access control disabled
all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
INF roles_query not defined: attribute based access control disabled
all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected
105048 10398 ns/op 18342 B/op 55 allocs/op
PASS
ok github.com/dosco/super-graph/core 1.328s
PASS
ok github.com/dosco/super-graph/core/internal/allow 0.088s
? github.com/dosco/super-graph/core/internal/crypto [no test files]
? github.com/dosco/super-graph/core/internal/integration_tests [no test files]
PASS
ok github.com/dosco/super-graph/core/internal/integration_tests/cockroachdb 0.121s
PASS
ok github.com/dosco/super-graph/core/internal/integration_tests/postgresql 0.118s
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/core/internal/psql
BenchmarkCompile-16 79845 14428 ns/op 4584 B/op 39 allocs/op
BenchmarkCompileParallel-16 326205 3918 ns/op 4633 B/op 39 allocs/op
PASS
ok github.com/dosco/super-graph/core/internal/psql 2.696s
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/core/internal/qcode
BenchmarkQCompile-16 146953 8049 ns/op 3756 B/op 28 allocs/op
BenchmarkQCompileP-16 475936 2447 ns/op 3790 B/op 28 allocs/op
BenchmarkParse-16 140811 8163 ns/op 3902 B/op 18 allocs/op
BenchmarkParseP-16 571345 2041 ns/op 3903 B/op 18 allocs/op
BenchmarkSchemaParse-16 230715 5012 ns/op 3968 B/op 57 allocs/op
BenchmarkSchemaParseP-16 802426 1565 ns/op 3968 B/op 57 allocs/op
PASS
ok github.com/dosco/super-graph/core/internal/qcode 8.427s
? github.com/dosco/super-graph/core/internal/util [no test files]

32
core/core.go
View File

@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
"hash/maphash"
"time"
"github.com/dosco/super-graph/core/internal/psql"
@ -165,32 +166,43 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
} else {
role = c.role
}
c.res.role = role
ps, ok := c.sg.prepared[stmtHash(c.res.name, role)]
h := maphash.Hash{}
h.SetSeed(c.sg.hashSeed)
q, ok := c.sg.queries[queryID(&h, c.res.name, role)]
if !ok {
return nil, nil, errNotFound
}
c.res.sql = ps.st.sql
if q.sd == nil {
q.Do(func() { c.sg.prepare(q, role) })
if q.err != nil {
return nil, nil, err
}
}
c.res.sql = q.st.sql
var root []byte
var row *sql.Row
varsList, err := c.argList(ps.st.md)
varsList, err := c.argList(q.st.md)
if err != nil {
return nil, nil, err
}
if useTx {
row = tx.Stmt(ps.sd).QueryRow(varsList...)
row = tx.Stmt(q.sd).QueryRow(varsList...)
} else {
row = ps.sd.QueryRow(varsList...)
row = q.sd.QueryRow(varsList...)
}
if ps.roleArg {
if q.roleArg {
err = row.Scan(&role, &root)
} else {
err = row.Scan(&root)
@ -204,15 +216,15 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
if useTx {
if err := tx.Commit(); err != nil {
return nil, nil, err
return nil, nil, q.err
}
}
if root, err = c.sg.encryptCursor(ps.st.qc, root); err != nil {
if root, err = c.sg.encryptCursor(q.st.qc, root); err != nil {
return nil, nil, err
}
return root, &ps.st, nil
return root, &q.st, nil
}
func (c *scontext) resolveSQL() ([]byte, *stmt, error) {

162
core/prepare.go
View File

@ -2,120 +2,97 @@ package core
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"fmt"
"hash/maphash"
"io"
"strings"
"sync"
"github.com/dosco/super-graph/core/internal/allow"
"github.com/dosco/super-graph/core/internal/qcode"
)
type preparedItem struct {
type query struct {
sync.Once
sd *sql.Stmt
ai allow.Item
qt qcode.QType
err error
st stmt
roleArg bool
}
func (sg *SuperGraph) initPrepared() error {
ct := context.Background()
func (sg *SuperGraph) prepare(q *query, role string) {
var stmts []stmt
var err error
qb := []byte(q.ai.Query)
switch q.qt {
case qcode.QTQuery:
if sg.abacEnabled {
stmts, err = sg.buildMultiStmt(qb, q.ai.Vars)
} else {
stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role)
}
case qcode.QTMutation:
stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role)
}
if err != nil {
sg.log.Printf("WRN %s %s: %v", q.qt, q.ai.Name, err)
}
q.st = stmts[0]
q.roleArg = len(stmts) > 1
q.sd, err = sg.db.Prepare(q.st.sql)
if err != nil {
q.err = fmt.Errorf("prepare failed: %v: %s", err, q.st.sql)
}
}
func (sg *SuperGraph) initPrepared() error {
if sg.allowList.IsPersist() {
return nil
}
sg.prepared = make(map[string]*preparedItem)
tx, err := sg.db.BeginTx(ct, nil)
if err != nil {
return err
}
defer tx.Rollback() //nolint: errcheck
if err = sg.prepareRoleStmt(tx); err != nil {
return fmt.Errorf("prepareRoleStmt: %w", err)
if err := sg.prepareRoleStmt(); err != nil {
return fmt.Errorf("role query: %w", err)
}
if err := tx.Commit(); err != nil {
return err
}
success := 0
sg.queries = make(map[uint64]*query)
list, err := sg.allowList.Load()
if err != nil {
return err
}
h := maphash.Hash{}
h.SetSeed(sg.hashSeed)
for _, v := range list {
if len(v.Query) == 0 {
continue
}
q := &query{ai: v, qt: qcode.GetQType(v.Query)}
err := sg.prepareStmt(v)
if err != nil {
return err
} else {
success++
}
}
switch q.qt {
case qcode.QTQuery:
sg.queries[queryID(&h, v.Name, "user")] = q
h.Reset()
sg.log.Printf("INF allow list: prepared %d / %d queries", success, len(list))
return nil
}
func (sg *SuperGraph) prepareStmt(item allow.Item) error {
query := item.Query
qb := []byte(query)
vars := item.Vars
qt := qcode.GetQType(query)
ct := context.Background()
switch qt {
case qcode.QTQuery:
var stmts1 []stmt
var err error
if sg.abacEnabled {
stmts1, err = sg.buildMultiStmt(qb, vars)
} else {
stmts1, err = sg.buildRoleStmt(qb, vars, "user")
}
if err == nil {
if err = sg.prepare(ct, stmts1, stmtHash(item.Name, "user")); err != nil {
return err
if sg.anonExists {
sg.queries[queryID(&h, v.Name, "anon")] = q
h.Reset()
}
} else {
sg.log.Printf("WRN query %s: %v", item.Name, err)
}
if sg.anonExists {
stmts2, err := sg.buildRoleStmt(qb, vars, "anon")
if err == nil {
if err = sg.prepare(ct, stmts2, stmtHash(item.Name, "anon")); err != nil {
return err
}
} else {
sg.log.Printf("WRN query %s: %v", item.Name, err)
}
}
case qcode.QTMutation:
for _, role := range sg.conf.Roles {
stmts, err := sg.buildRoleStmt(qb, vars, role.Name)
if err == nil {
if err = sg.prepare(ct, stmts, stmtHash(item.Name, role.Name)); err != nil {
return err
}
} else {
sg.log.Printf("WRN mutation %s: %v", item.Name, err)
case qcode.QTMutation:
for _, role := range sg.conf.Roles {
sg.queries[queryID(&h, v.Name, role.Name)] = q
h.Reset()
}
}
}
@ -123,22 +100,8 @@ func (sg *SuperGraph) prepareStmt(item allow.Item) error {
return nil
}
func (sg *SuperGraph) prepare(ct context.Context, st []stmt, key string) error {
sd, err := sg.db.PrepareContext(ct, st[0].sql)
if err != nil {
return fmt.Errorf("prepare failed: %v: %s", err, st[0].sql)
}
sg.prepared[key] = &preparedItem{
sd: sd,
st: st[0],
roleArg: len(st) > 1,
}
return nil
}
// nolint: errcheck
func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error {
func (sg *SuperGraph) prepareRoleStmt() error {
var err error
if !sg.abacEnabled {
@ -169,7 +132,7 @@ func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error {
io.WriteString(w, `) AS "_sg_auth_roles_query" LIMIT 1) `)
io.WriteString(w, `ELSE 'anon' END) FROM (VALUES (1)) AS "_sg_auth_filler" LIMIT 1; `)
sg.getRole, err = tx.Prepare(w.String())
sg.getRole, err = sg.db.Prepare(w.String())
if err != nil {
return err
}
@ -200,9 +163,8 @@ func (sg *SuperGraph) initAllowList() error {
}
// nolint: errcheck
func stmtHash(name string, role string) string {
h := sha256.New()
io.WriteString(h, strings.ToLower(name))
io.WriteString(h, role)
return hex.EncodeToString(h.Sum(nil))
func queryID(h *maphash.Hash, name string, role string) uint64 {
h.WriteString(name)
h.WriteString(role)
return h.Sum64()
}

31
internal/serv/internal/migrate/migrate.go
View File

@ -6,9 +6,11 @@ import (
"database/sql"
"fmt"
"io/ioutil"
"log"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
"strings"
"text/template"
@ -105,39 +107,40 @@ func (defaultMigratorFS) Glob(pattern string) ([]string, error) {
func FindMigrationsEx(path string, fs MigratorFS) ([]string, error) {
path = strings.TrimRight(path, string(filepath.Separator))
fileInfos, err := fs.ReadDir(path)
files, err := ioutil.ReadDir(path)
if err != nil {
return nil, err
log.Fatal(err)
}
paths := make([]string, 0, len(fileInfos))
for _, fi := range fileInfos {
fm := make(map[int]string, len(files))
keys := make([]int, 0, len(files))
for _, fi := range files {
if fi.IsDir() {
continue
}
matches := migrationPattern.FindStringSubmatch(fi.Name())
if len(matches) != 2 {
continue
}
n, err := strconv.ParseInt(matches[1], 10, 32)
n, err := strconv.Atoi(matches[1])
if err != nil {
// The regexp already validated that the prefix is all digits so this *should* never fail
return nil, err
}
mcount := len(paths)
fm[n] = filepath.Join(path, fi.Name())
keys = append(keys, n)
}
if n < int64(mcount) {
return nil, fmt.Errorf("Duplicate migration %d", n)
}
sort.Ints(keys)
if int64(mcount) < n {
return nil, fmt.Errorf("Missing migration %d", mcount)
}
paths = append(paths, filepath.Join(path, fi.Name()))
paths := make([]string, 0, len(keys))
for _, k := range keys {
paths = append(paths, fm[k])
}
return paths, nil