diff --git a/core/api.go b/core/api.go index e4740d3..e4437f6 100644 --- a/core/api.go +++ b/core/api.go @@ -49,6 +49,7 @@ import ( "crypto/sha256" "database/sql" "encoding/json" + "hash/maphash" _log "log" "os" @@ -83,7 +84,8 @@ type SuperGraph struct { schema *psql.DBSchema allowList *allow.List encKey [32]byte - prepared map[string]*preparedItem + hashSeed maphash.Seed + queries map[uint64]*query roles map[string]*Role getRole *sql.Stmt rmap map[uint64]*resolvFn @@ -107,10 +109,11 @@ func newSuperGraph(conf *Config, db *sql.DB, dbinfo *psql.DBInfo) (*SuperGraph, } sg := &SuperGraph{ - conf: conf, - db: db, - dbinfo: dbinfo, - log: _log.New(os.Stdout, "", 0), + conf: conf, + db: db, + dbinfo: dbinfo, + log: _log.New(os.Stdout, "", 0), + hashSeed: maphash.MakeSeed(), } if err := sg.initConfig(); err != nil { diff --git a/core/bench.11 b/core/bench.11 new file mode 100644 index 0000000..95233cd --- /dev/null +++ b/core/bench.11 @@ -0,0 +1,41 @@ +INF roles_query not defined: attribute based access control disabled +all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/core +BenchmarkGraphQL-16 INF roles_query not defined: attribute based access control disabled +all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected +INF roles_query not defined: attribute based access control disabled +all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected +INF roles_query not defined: attribute based access control disabled +all expectations were already fulfilled, call to Query 'SELECT jsonb_build_object('users', "__sj_0"."json", 'products', "__sj_1"."json") as "__root" FROM (SELECT coalesce(jsonb_agg("__sj_1"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_1".*) AS "json"FROM (SELECT "products_1"."id" AS "id", "products_1"."name" AS "name", "__sj_2"."json" AS "customers", "__sj_3"."json" AS "user" FROM (SELECT "products"."id", "products"."name", "products"."user_id" FROM "products" LIMIT ('20') :: integer) AS "products_1" LEFT OUTER JOIN LATERAL (SELECT to_jsonb("__sr_3".*) AS "json"FROM (SELECT "users_3"."full_name" AS "full_name", "users_3"."phone" AS "phone", "users_3"."email" AS "email" FROM (SELECT "users"."full_name", "users"."phone", "users"."email" FROM "users" WHERE ((("users"."id") = ("products_1"."user_id"))) LIMIT ('1') :: integer) AS "users_3") AS "__sr_3") AS "__sj_3" ON ('true') LEFT OUTER JOIN LATERAL (SELECT coalesce(jsonb_agg("__sj_2"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_2".*) AS "json"FROM (SELECT "customers_2"."id" AS "id", "customers_2"."email" AS "email" FROM (SELECT "customers"."id", "customers"."email" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_1"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_2") AS "__sr_2") AS "__sj_2") AS "__sj_2" ON ('true')) AS "__sr_1") AS "__sj_1") AS "__sj_1", (SELECT coalesce(jsonb_agg("__sj_0"."json"), '[]') as "json" FROM (SELECT to_jsonb("__sr_0".*) AS "json"FROM (SELECT "users_0"."id" AS "id", "users_0"."name" AS "name" FROM (SELECT "users"."id" FROM "users" GROUP BY "users"."id" LIMIT ('20') :: integer) AS "users_0") AS "__sr_0") AS "__sj_0") AS "__sj_0"' with args [] was not expected + 105048 10398 ns/op 18342 B/op 55 allocs/op +PASS +ok github.com/dosco/super-graph/core 1.328s +PASS +ok github.com/dosco/super-graph/core/internal/allow 0.088s +? github.com/dosco/super-graph/core/internal/crypto [no test files] +? github.com/dosco/super-graph/core/internal/integration_tests [no test files] +PASS +ok github.com/dosco/super-graph/core/internal/integration_tests/cockroachdb 0.121s +PASS +ok github.com/dosco/super-graph/core/internal/integration_tests/postgresql 0.118s +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/core/internal/psql +BenchmarkCompile-16 79845 14428 ns/op 4584 B/op 39 allocs/op +BenchmarkCompileParallel-16 326205 3918 ns/op 4633 B/op 39 allocs/op +PASS +ok github.com/dosco/super-graph/core/internal/psql 2.696s +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/core/internal/qcode +BenchmarkQCompile-16 146953 8049 ns/op 3756 B/op 28 allocs/op +BenchmarkQCompileP-16 475936 2447 ns/op 3790 B/op 28 allocs/op +BenchmarkParse-16 140811 8163 ns/op 3902 B/op 18 allocs/op +BenchmarkParseP-16 571345 2041 ns/op 3903 B/op 18 allocs/op +BenchmarkSchemaParse-16 230715 5012 ns/op 3968 B/op 57 allocs/op +BenchmarkSchemaParseP-16 802426 1565 ns/op 3968 B/op 57 allocs/op +PASS +ok github.com/dosco/super-graph/core/internal/qcode 8.427s +? github.com/dosco/super-graph/core/internal/util [no test files] diff --git a/core/core.go b/core/core.go index 8ebd6a7..3f01004 100644 --- a/core/core.go +++ b/core/core.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "hash/maphash" "time" "github.com/dosco/super-graph/core/internal/psql" @@ -165,32 +166,43 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) { } else { role = c.role - } c.res.role = role - ps, ok := c.sg.prepared[stmtHash(c.res.name, role)] + h := maphash.Hash{} + h.SetSeed(c.sg.hashSeed) + + q, ok := c.sg.queries[queryID(&h, c.res.name, role)] if !ok { return nil, nil, errNotFound } - c.res.sql = ps.st.sql + + if q.sd == nil { + q.Do(func() { c.sg.prepare(q, role) }) + + if q.err != nil { + return nil, nil, err + } + } + + c.res.sql = q.st.sql var root []byte var row *sql.Row - varsList, err := c.argList(ps.st.md) + varsList, err := c.argList(q.st.md) if err != nil { return nil, nil, err } if useTx { - row = tx.Stmt(ps.sd).QueryRow(varsList...) + row = tx.Stmt(q.sd).QueryRow(varsList...) } else { - row = ps.sd.QueryRow(varsList...) + row = q.sd.QueryRow(varsList...) } - if ps.roleArg { + if q.roleArg { err = row.Scan(&role, &root) } else { err = row.Scan(&root) @@ -204,15 +216,15 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) { if useTx { if err := tx.Commit(); err != nil { - return nil, nil, err + return nil, nil, q.err } } - if root, err = c.sg.encryptCursor(ps.st.qc, root); err != nil { + if root, err = c.sg.encryptCursor(q.st.qc, root); err != nil { return nil, nil, err } - return root, &ps.st, nil + return root, &q.st, nil } func (c *scontext) resolveSQL() ([]byte, *stmt, error) { diff --git a/core/prepare.go b/core/prepare.go index ac459eb..d91ab23 100644 --- a/core/prepare.go +++ b/core/prepare.go @@ -2,120 +2,97 @@ package core import ( "bytes" - "context" - "crypto/sha256" "database/sql" - "encoding/hex" "fmt" + "hash/maphash" "io" "strings" + "sync" "github.com/dosco/super-graph/core/internal/allow" "github.com/dosco/super-graph/core/internal/qcode" ) -type preparedItem struct { +type query struct { + sync.Once sd *sql.Stmt + ai allow.Item + qt qcode.QType + err error st stmt roleArg bool } -func (sg *SuperGraph) initPrepared() error { - ct := context.Background() +func (sg *SuperGraph) prepare(q *query, role string) { + var stmts []stmt + var err error + qb := []byte(q.ai.Query) + + switch q.qt { + case qcode.QTQuery: + if sg.abacEnabled { + stmts, err = sg.buildMultiStmt(qb, q.ai.Vars) + } else { + stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role) + } + + case qcode.QTMutation: + stmts, err = sg.buildRoleStmt(qb, q.ai.Vars, role) + } + + if err != nil { + sg.log.Printf("WRN %s %s: %v", q.qt, q.ai.Name, err) + } + + q.st = stmts[0] + q.roleArg = len(stmts) > 1 + + q.sd, err = sg.db.Prepare(q.st.sql) + if err != nil { + q.err = fmt.Errorf("prepare failed: %v: %s", err, q.st.sql) + } +} + +func (sg *SuperGraph) initPrepared() error { if sg.allowList.IsPersist() { return nil } - sg.prepared = make(map[string]*preparedItem) - tx, err := sg.db.BeginTx(ct, nil) - if err != nil { - return err - } - defer tx.Rollback() //nolint: errcheck - - if err = sg.prepareRoleStmt(tx); err != nil { - return fmt.Errorf("prepareRoleStmt: %w", err) + if err := sg.prepareRoleStmt(); err != nil { + return fmt.Errorf("role query: %w", err) } - if err := tx.Commit(); err != nil { - return err - } - - success := 0 + sg.queries = make(map[uint64]*query) list, err := sg.allowList.Load() if err != nil { return err } + h := maphash.Hash{} + h.SetSeed(sg.hashSeed) + for _, v := range list { if len(v.Query) == 0 { continue } + q := &query{ai: v, qt: qcode.GetQType(v.Query)} - err := sg.prepareStmt(v) - if err != nil { - return err - } else { - success++ - } - } + switch q.qt { + case qcode.QTQuery: + sg.queries[queryID(&h, v.Name, "user")] = q + h.Reset() - sg.log.Printf("INF allow list: prepared %d / %d queries", success, len(list)) - - return nil -} - -func (sg *SuperGraph) prepareStmt(item allow.Item) error { - query := item.Query - qb := []byte(query) - vars := item.Vars - - qt := qcode.GetQType(query) - ct := context.Background() - - switch qt { - case qcode.QTQuery: - var stmts1 []stmt - var err error - - if sg.abacEnabled { - stmts1, err = sg.buildMultiStmt(qb, vars) - } else { - stmts1, err = sg.buildRoleStmt(qb, vars, "user") - } - - if err == nil { - if err = sg.prepare(ct, stmts1, stmtHash(item.Name, "user")); err != nil { - return err + if sg.anonExists { + sg.queries[queryID(&h, v.Name, "anon")] = q + h.Reset() } - } else { - sg.log.Printf("WRN query %s: %v", item.Name, err) - } - if sg.anonExists { - stmts2, err := sg.buildRoleStmt(qb, vars, "anon") - - if err == nil { - if err = sg.prepare(ct, stmts2, stmtHash(item.Name, "anon")); err != nil { - return err - } - } else { - sg.log.Printf("WRN query %s: %v", item.Name, err) - } - } - - case qcode.QTMutation: - for _, role := range sg.conf.Roles { - stmts, err := sg.buildRoleStmt(qb, vars, role.Name) - - if err == nil { - if err = sg.prepare(ct, stmts, stmtHash(item.Name, role.Name)); err != nil { - return err - } - } else { - sg.log.Printf("WRN mutation %s: %v", item.Name, err) + case qcode.QTMutation: + for _, role := range sg.conf.Roles { + sg.queries[queryID(&h, v.Name, role.Name)] = q + h.Reset() } } } @@ -123,22 +100,8 @@ func (sg *SuperGraph) prepareStmt(item allow.Item) error { return nil } -func (sg *SuperGraph) prepare(ct context.Context, st []stmt, key string) error { - sd, err := sg.db.PrepareContext(ct, st[0].sql) - if err != nil { - return fmt.Errorf("prepare failed: %v: %s", err, st[0].sql) - } - - sg.prepared[key] = &preparedItem{ - sd: sd, - st: st[0], - roleArg: len(st) > 1, - } - return nil -} - // nolint: errcheck -func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error { +func (sg *SuperGraph) prepareRoleStmt() error { var err error if !sg.abacEnabled { @@ -169,7 +132,7 @@ func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error { io.WriteString(w, `) AS "_sg_auth_roles_query" LIMIT 1) `) io.WriteString(w, `ELSE 'anon' END) FROM (VALUES (1)) AS "_sg_auth_filler" LIMIT 1; `) - sg.getRole, err = tx.Prepare(w.String()) + sg.getRole, err = sg.db.Prepare(w.String()) if err != nil { return err } @@ -200,9 +163,8 @@ func (sg *SuperGraph) initAllowList() error { } // nolint: errcheck -func stmtHash(name string, role string) string { - h := sha256.New() - io.WriteString(h, strings.ToLower(name)) - io.WriteString(h, role) - return hex.EncodeToString(h.Sum(nil)) +func queryID(h *maphash.Hash, name string, role string) uint64 { + h.WriteString(name) + h.WriteString(role) + return h.Sum64() } diff --git a/internal/serv/internal/migrate/migrate.go b/internal/serv/internal/migrate/migrate.go index 163a386..207c255 100644 --- a/internal/serv/internal/migrate/migrate.go +++ b/internal/serv/internal/migrate/migrate.go @@ -6,9 +6,11 @@ import ( "database/sql" "fmt" "io/ioutil" + "log" "os" "path/filepath" "regexp" + "sort" "strconv" "strings" "text/template" @@ -105,39 +107,40 @@ func (defaultMigratorFS) Glob(pattern string) ([]string, error) { func FindMigrationsEx(path string, fs MigratorFS) ([]string, error) { path = strings.TrimRight(path, string(filepath.Separator)) - fileInfos, err := fs.ReadDir(path) + files, err := ioutil.ReadDir(path) if err != nil { - return nil, err + log.Fatal(err) } - paths := make([]string, 0, len(fileInfos)) - for _, fi := range fileInfos { + fm := make(map[int]string, len(files)) + keys := make([]int, 0, len(files)) + + for _, fi := range files { if fi.IsDir() { continue } matches := migrationPattern.FindStringSubmatch(fi.Name()) + if len(matches) != 2 { continue } - n, err := strconv.ParseInt(matches[1], 10, 32) + n, err := strconv.Atoi(matches[1]) if err != nil { // The regexp already validated that the prefix is all digits so this *should* never fail return nil, err } - mcount := len(paths) + fm[n] = filepath.Join(path, fi.Name()) + keys = append(keys, n) + } - if n < int64(mcount) { - return nil, fmt.Errorf("Duplicate migration %d", n) - } + sort.Ints(keys) - if int64(mcount) < n { - return nil, fmt.Errorf("Missing migration %d", mcount) - } - - paths = append(paths, filepath.Join(path, fi.Name())) + paths := make([]string, 0, len(keys)) + for _, k := range keys { + paths = append(paths, fm[k]) } return paths, nil