wpetit
/
super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make remote joins use parallel http requests

This commit is contained in:
Vikram Rangnekar 2019-06-02 01:38:51 -04:00
parent f9fc5dd7de
commit 8b06473e58
10 changed files with 143 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
docs/guide.md
View File

@ -345,13 +345,17 @@ class AddSearchColumn < ActiveRecord::Migration[5.1]
end end
``` ```
## Join with remote REST APIs ## Remote Joins
It often happens that after fetching some data from the DB we need to call another API to fetch some more data and all this combined into a single JSON response. For example along with a list of users you need their last 5 payments from Stripe. This requires you to query your DB for the users and Stripe for the payments. Super Graph handles all this for you also only the fields you requested from the Stripe API are returned. All this is a very performance focused way. It often happens that after fetching some data from the DB we need to call another API to fetch some more data and all this combined into a single JSON response. For example along with a list of users you need their last 5 payments from Stripe. This requires you to query your DB for the users and Stripe for the payments. Super Graph handles all this for you also only the fields you requested from the Stripe API are returned.
::: tip Is this fast?
Super Graph is able fetch remote data and merge it with the DB response in an efficient manner. Several optimizations such as parallel HTTP requests and a zero-allocation JSON merge algorithm makes this very fast. All of this without you having to write a line of code.
:::
For example you need to list the last 3 payments made by a user. You will first need to look up the user in the database and then call the Stripe API to fetch his last 3 payments. For this to work your user table in the db has a `customer_id` column that contains his Stripe customer ID. For example you need to list the last 3 payments made by a user. You will first need to look up the user in the database and then call the Stripe API to fetch his last 3 payments. For this to work your user table in the db has a `customer_id` column that contains his Stripe customer ID.
Similiarly you might also have the need to fetch the users last tweet and include that too. Super Graph can handle this for you using it's `API Stitching` feature. Similiarly you could also fetch the users last tweet, lead info from Salesforce or whatever else you need. It's fine to mix up several different `remote joins` into a single GraphQL query.
### Stripe API example ### Stripe API example

5
psql/bench.0
View File

@ -1,6 +1,7 @@
goos: darwin goos: darwin
goarch: amd64 goarch: amd64
pkg: github.com/dosco/super-graph/psql pkg: github.com/dosco/super-graph/psql
BenchmarkCompileGQLToSQL-8 50000 38882 ns/op 15177 B/op 266 allocs/op BenchmarkCompile-8 50000 38531 ns/op 5175 B/op 148 allocs/op
BenchmarkCompileParallel-8 200000 11472 ns/op 5237 B/op 148 allocs/op
PASS PASS
ok github.com/dosco/super-graph/psql 2.473s ok github.com/dosco/super-graph/psql 4.686s

6
psql/bench.1
View File

@ -1,6 +0,0 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompileGQLToSQL-8 50000 39601 ns/op 20165 B/op 263 allocs/op
PASS
ok github.com/dosco/super-graph/psql 2.549s

6
psql/bench.2
View File

@ -1,6 +0,0 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompileGQLToSQL-8 50000 35559 ns/op 8453 B/op 228 allocs/op
PASS
ok github.com/dosco/super-graph/psql 2.162s

6
psql/bench.3
View File

@ -1,6 +0,0 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompileGQLToSQL-8 50000 28320 ns/op 7698 B/op 154 allocs/op
PASS
ok github.com/dosco/super-graph/psql 1.724s

22
psql/psql.go
View File

@ -91,7 +91,7 @@ func (c *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) {
ignored |= skipped ignored |= skipped
for _, id := range v.sel.Children { for _, id := range v.sel.Children {
if hasBit(skipped, id) { if hasBit(skipped, uint16(id)) {
continue continue
} }
child := &qc.Query.Selects[id] child := &qc.Query.Selects[id]
@ -377,7 +377,7 @@ func (v *selectBlock) renderJoinedColumns(w *bytes.Buffer, skipped uint32) error
colsRendered := len(v.sel.Cols) != 0 colsRendered := len(v.sel.Cols) != 0
for _, id := range v.sel.Children { for _, id := range v.sel.Children {
skipThis := hasBit(skipped, id) skipThis := hasBit(skipped, uint16(id))
if colsRendered && !skipThis { if colsRendered && !skipThis {
io.WriteString(w, ", ") io.WriteString(w, ", ")
@ -884,7 +884,7 @@ func alias(w *bytes.Buffer, alias string) {
w.WriteString(`"`) w.WriteString(`"`)
} }
func aliasWithID(w *bytes.Buffer, alias string, id uint16) { func aliasWithID(w *bytes.Buffer, alias string, id int16) {
w.WriteString(` AS "`) w.WriteString(` AS "`)
w.WriteString(alias) w.WriteString(alias)
w.WriteString(`_`) w.WriteString(`_`)
@ -892,7 +892,7 @@ func aliasWithID(w *bytes.Buffer, alias string, id uint16) {
w.WriteString(`"`) w.WriteString(`"`)
} }
func aliasWithIDSuffix(w *bytes.Buffer, alias string, id uint16, suffix string) { func aliasWithIDSuffix(w *bytes.Buffer, alias string, id int16, suffix string) {
w.WriteString(` AS "`) w.WriteString(` AS "`)
w.WriteString(alias) w.WriteString(alias)
w.WriteString(`_`) w.WriteString(`_`)
@ -917,7 +917,7 @@ func colWithTable(w *bytes.Buffer, table, col string) {
w.WriteString(`"`) w.WriteString(`"`)
} }
func colWithTableID(w *bytes.Buffer, table string, id uint16, col string) { func colWithTableID(w *bytes.Buffer, table string, id int16, col string) {
w.WriteString(`"`) w.WriteString(`"`)
w.WriteString(table) w.WriteString(table)
w.WriteString(`_`) w.WriteString(`_`)
@ -927,7 +927,7 @@ func colWithTableID(w *bytes.Buffer, table string, id uint16, col string) {
w.WriteString(`"`) w.WriteString(`"`)
} }
func colWithTableIDAlias(w *bytes.Buffer, table string, id uint16, col, alias string) { func colWithTableIDAlias(w *bytes.Buffer, table string, id int16, col, alias string) {
w.WriteString(`"`) w.WriteString(`"`)
w.WriteString(table) w.WriteString(table)
w.WriteString(`_`) w.WriteString(`_`)
@ -939,7 +939,7 @@ func colWithTableIDAlias(w *bytes.Buffer, table string, id uint16, col, alias st
w.WriteString(`"`) w.WriteString(`"`)
} }
func colWithTableIDSuffixAlias(w *bytes.Buffer, table string, id uint16, func colWithTableIDSuffixAlias(w *bytes.Buffer, table string, id int16,
suffix, col, alias string) { suffix, col, alias string) {
w.WriteString(`"`) w.WriteString(`"`)
w.WriteString(table) w.WriteString(table)
@ -953,7 +953,7 @@ func colWithTableIDSuffixAlias(w *bytes.Buffer, table string, id uint16,
w.WriteString(`"`) w.WriteString(`"`)
} }
func tableIDColSuffix(w *bytes.Buffer, table string, id uint16, col, suffix string) { func tableIDColSuffix(w *bytes.Buffer, table string, id int16, col, suffix string) {
w.WriteString(`"`) w.WriteString(`"`)
w.WriteString(table) w.WriteString(table)
w.WriteString(`_`) w.WriteString(`_`)
@ -966,18 +966,18 @@ func tableIDColSuffix(w *bytes.Buffer, table string, id uint16, col, suffix stri
const charset = "0123456789" const charset = "0123456789"
func int2string(w *bytes.Buffer, val uint16) { func int2string(w *bytes.Buffer, val int16) {
if val < 10 { if val < 10 {
w.WriteByte(charset[val]) w.WriteByte(charset[val])
return return
} }
temp := uint16(0) temp := int16(0)
val2 := val val2 := val
for val2 > 0 { for val2 > 0 {
temp *= 10 temp *= 10
temp += val2 % 10 temp += val2 % 10
val2 = uint16(math.Floor(float64(val2 / 10))) val2 = int16(math.Floor(float64(val2 / 10)))
} }
val3 := temp val3 := temp

37
psql/psql_test.go
View File

@ -480,8 +480,7 @@ func TestCompileGQL(t *testing.T) {
t.Run("syntheticTables", syntheticTables) t.Run("syntheticTables", syntheticTables)
} }
func BenchmarkCompileGQLToSQL(b *testing.B) { var benchGQL = `query {
gql := `query {
products( products(
# returns only 30 items # returns only 30 items
limit: 30, limit: 30,
@ -502,23 +501,47 @@ func BenchmarkCompileGQLToSQL(b *testing.B) {
picture : avatar picture : avatar
} }
} }
}` }`
w := &bytes.Buffer() func BenchmarkCompile(b *testing.B) {
w := &bytes.Buffer{}
b.ResetTimer() b.ResetTimer()
b.ReportAllocs() b.ReportAllocs()
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {
qc, err := qcompile.CompileQuery(gql) w.Reset()
qc, err := qcompile.CompileQuery(benchGQL)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
_, sqlStmt, err := pcompile.Compile(qc, w) _, err = pcompile.Compile(qc, w)
if err != nil { if err != nil {
b.Fatal(err) b.Fatal(err)
} }
w.Reset()
} }
} }
func BenchmarkCompileParallel(b *testing.B) {
b.ReportAllocs()
b.RunParallel(func(pb *testing.PB) {
w := &bytes.Buffer{}
for pb.Next() {
w.Reset()
qc, err := qcompile.CompileQuery(benchGQL)
if err != nil {
b.Fatal(err)
}
_, err = pcompile.Compile(qc, w)
if err != nil {
b.Fatal(err)
}
}
})
}

12
qcode/parse.go
View File

@ -48,14 +48,14 @@ func (o *Operation) Reset() {
} }
type Field struct { type Field struct {
ID uint16 ID int16
Name string Name string
Alias string Alias string
Args []Arg Args []Arg
argsA [10]Arg argsA [10]Arg
ParentID uint16 ParentID int16
Children []uint16 Children []int16
childrenA [10]uint16 childrenA [10]int16
} }
type Arg struct { type Arg struct {
@ -277,7 +277,7 @@ func (p *Parser) parseFields(fields []Field) ([]Field, error) {
return nil, errors.New("expecting an alias or field name") return nil, errors.New("expecting an alias or field name")
} }
fields = append(fields, Field{ID: uint16(len(fields))}) fields = append(fields, Field{ID: int16(len(fields))})
f := &fields[(len(fields) - 1)] f := &fields[(len(fields) - 1)]
f.Args = f.argsA[:0] f.Args = f.argsA[:0]
f.Children = f.childrenA[:0] f.Children = f.childrenA[:0]
@ -288,7 +288,7 @@ func (p *Parser) parseFields(fields []Field) ([]Field, error) {
if f.ID != 0 { if f.ID != 0 {
intf := st.Peek() intf := st.Peek()
pid, ok := intf.(uint16) pid, ok := intf.(int16)
if !ok { if !ok {
return nil, fmt.Errorf("14: unexpected value %v (%t)", intf, intf) return nil, fmt.Errorf("14: unexpected value %v (%t)", intf, intf)

13
qcode/qcode.go
View File

@ -29,8 +29,8 @@ type Column struct {
} }
type Select struct { type Select struct {
ID uint16 ID int16
ParentID uint16 ParentID int16
RelID uint64 RelID uint64
Args map[string]*Node Args map[string]*Node
AsList bool AsList bool
@ -42,7 +42,7 @@ type Select struct {
OrderBy []*OrderBy OrderBy []*OrderBy
DistinctOn []string DistinctOn []string
Paging Paging Paging Paging
Children []uint16 Children []int16
} }
type Exp struct { type Exp struct {
@ -197,7 +197,8 @@ func (com *Compiler) CompileQuery(query string) (*QCode, error) {
} }
func (com *Compiler) compileQuery(op *Operation) (*Query, error) { func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
var id, parentID uint16 id := int16(0)
parentID := int16(-1)
selects := make([]Select, 0, 5) selects := make([]Select, 0, 5)
st := util.NewStack() st := util.NewStack()
@ -218,7 +219,7 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
} }
intf := st.Pop() intf := st.Pop()
fid, ok := intf.(uint16) fid, ok := intf.(int16)
if !ok { if !ok {
return nil, fmt.Errorf("15: unexpected value %v (%t)", intf, intf) return nil, fmt.Errorf("15: unexpected value %v (%t)", intf, intf)
@ -235,7 +236,7 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
ID: id, ID: id,
ParentID: parentID, ParentID: parentID,
Table: tn, Table: tn,
Children: make([]uint16, 0, 5), Children: make([]int16, 0, 5),
} }
if s.ID != 0 { if s.ID != 0 {

64
serv/core.go
View File

@ -8,6 +8,7 @@ import (
"io" "io"
"net/http" "net/http"
"os" "os"
"sync"
"time" "time"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
@ -74,10 +75,6 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
return errors.New("something wrong no remote ids found in db response") return errors.New("something wrong no remote ids found in db response")
} }
if err != nil {
return err
}
var ob bytes.Buffer var ob bytes.Buffer
err = jsn.Replace(&ob, data, from, to) err = jsn.Replace(&ob, data, from, to)
@ -131,7 +128,7 @@ func (c *coreContext) resolveRemote(
} }
if conf.EnableTracing { if conf.EnableTracing {
c.addTrace(s, st) c.addTrace(sel, s.ID, st)
} }
if len(r.Path) != 0 { if len(r.Path) != 0 {
@ -163,9 +160,14 @@ func (c *coreContext) resolveRemotes(
// replacement data for the marked insertion points // replacement data for the marked insertion points
// key and value will be replaced by whats below // key and value will be replaced by whats below
to := make([]jsn.Field, 0, len(from)) to := make([]jsn.Field, len(from))
for _, id := range from { var wg sync.WaitGroup
wg.Add(len(from))
var cerr error
for i, id := range from {
// use the json key to find the related Select object // use the json key to find the related Select object
k1 := xxhash.Sum64(id.Key) k1 := xxhash.Sum64(id.Key)
@ -190,15 +192,19 @@ func (c *coreContext) resolveRemotes(
return nil, nil return nil, nil
} }
go func(n int) {
defer wg.Done()
st := time.Now() st := time.Now()
b, err := r.Fn(req, id) b, err := r.Fn(req, id)
if err != nil { if err != nil {
return nil, err cerr = err
return
} }
if conf.EnableTracing { if conf.EnableTracing {
c.addTrace(s, st) c.addTrace(sel, s.ID, st)
} }
if len(r.Path) != 0 { if len(r.Path) != 0 {
@ -210,19 +216,24 @@ func (c *coreContext) resolveRemotes(
if len(s.Cols) != 0 { if len(s.Cols) != 0 {
err = jsn.Filter(&ob, b, colsToList(s.Cols)) err = jsn.Filter(&ob, b, colsToList(s.Cols))
if err != nil { if err != nil {
return nil, err cerr = err
return
} }
} else { } else {
ob.WriteString("null") ob.WriteString("null")
} }
to = append(to, jsn.Field{[]byte(s.FieldName), ob.Bytes()}) to[n] = jsn.Field{[]byte(s.FieldName), ob.Bytes()}
}(i)
} }
return to, nil wg.Wait()
return to, cerr
} }
func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) ([]byte, uint32, error) { func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) (
[]byte, uint32, error) {
// var entry []byte // var entry []byte
// var key string // var key string
@ -291,7 +302,10 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) ([]byte, uint3
} }
if conf.EnableTracing && len(qc.Query.Selects) != 0 { if conf.EnableTracing && len(qc.Query.Selects) != 0 {
c.addTrace(&qc.Query.Selects[0], st) c.addTrace(
qc.Query.Selects,
qc.Query.Selects[0].ID,
st)
} }
return []byte(root), skipped, nil return []byte(root), skipped, nil
@ -302,7 +316,7 @@ func (c *coreContext) render(w io.Writer, data []byte) error {
return json.NewEncoder(w).Encode(c.res) return json.NewEncoder(w).Encode(c.res)
} }
func (c *coreContext) addTrace(sel *qcode.Select, st time.Time) { func (c *coreContext) addTrace(sel []qcode.Select, id int16, st time.Time) {
et := time.Now() et := time.Now()
du := et.Sub(st) du := et.Sub(st)
@ -317,10 +331,22 @@ func (c *coreContext) addTrace(sel *qcode.Select, st time.Time) {
c.res.Extensions.Tracing.EndTime = et c.res.Extensions.Tracing.EndTime = et
c.res.Extensions.Tracing.Duration = du c.res.Extensions.Tracing.Duration = du
n := 0
for i := id; i != -1; i = sel[i].ParentID {
n++
}
path := make([]string, n)
n--
for i := id; i != -1; i = sel[i].ParentID {
path[n] = sel[i].Table
n--
}
tr := resolver{ tr := resolver{
Path: []string{sel.Table}, Path: path,
ParentType: "Query", ParentType: "Query",
FieldName: sel.Table, FieldName: sel[id].Table,
ReturnType: "object", ReturnType: "object",
StartOffset: 1, StartOffset: 1,
Duration: du, Duration: du,
@ -337,7 +363,7 @@ func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) (
c := 0 c := 0
for i := range sel { for i := range sel {
s := &sel[i] s := &sel[i]
if isSkipped(skipped, s.ID) { if isSkipped(skipped, uint16(s.ID)) {
c++ c++
} }
} }
@ -354,7 +380,7 @@ func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) (
for i := range sel { for i := range sel {
s := &sel[i] s := &sel[i]
if isSkipped(skipped, s.ID) == false { if isSkipped(skipped, uint16(s.ID)) == false {
continue continue
} }