wpetit/super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add tracing for API stitching

Browse Source
This commit is contained in:
Vikram Rangnekar 2019-05-13 00:05:08 -04:00
parent f16e95ef22
commit e1d3bb9055
12 changed files with 160 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
config/dev.yml
View File

@ -110,14 +110,14 @@ database:
remotes:
- name: payments
id: stripe_id
path: data
pass_headers:
- cookie
- host
# set_headers:
# - name: authorize
# value: Bearer 1234567890
url: http://rails_app:3000/stripe/$id
path: data
# pass_headers:
# - cookie
# - host
set_headers:
- name: Authorization
value: Bearer <stripe_api_key>
- # You can create new fields that have a
# real db table backing them

12
config/prod.yml
View File

@ -105,6 +105,18 @@ database:
# even defaults.filter
filter: none
# remotes:
# - name: payments
# id: stripe_id
# url: http://rails_app:3000/stripe/$id
# path: data
# # pass_headers:
# # - cookie
# # - host
# set_headers:
# - name: Authorization
# value: Bearer <stripe_api_key>
- # You can create new fields that have a
# real db table backing them
name: me

BIN
docs/.vuepress/public/tracing.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 42 KiB

48
docs/guide.md
View File

@ -347,7 +347,7 @@ class AddSearchColumn < ActiveRecord::Migration[5.1]
end
```
## Stitching in REST APIs
## API Stitching
It often happens that after fetching some data from the DB we need to call another API to fetch some more data and all this combined into a single JSON response.
@ -355,27 +355,26 @@ For example you need to list the last 3 payments made by a user. You will first
Similiarly you might also have the need to fetch the users last tweet and include that too. Super Graph can handle this for you using it's `API Stitching` feature.
### API Stitching configuration
### Stripe API example
The configuration is self explanatory. A `payments` field has been added under the `customers` table. This field is added to the `remotes` subsection that defines fields associated with `customers` that are remote and not real database columns.
The `id` parameter maps a column from the `customers` table to the `$id` variable. In this case it maps `$id` to the `customer_id` column.
```yaml
tables:
- name: customers
remotes:
- name: payments
id: customer_id
path: data
pass_headers:
- cookie
- host
# set_headers:
# - name: authorize
# value: Bearer 1234567890
url: http://rails_app:3000/stripe/$id
tables:
- name: customers
remotes:
- name: payments
id: stripe_id
url: http://rails_app:3000/stripe/$id
path: data
# pass_headers:
# - cookie
# - host
set_headers:
- name: Authorization
value: Bearer <stripe_api_key>
```
#### How do I make use of this?
@ -416,6 +415,11 @@ And voila here is the result. You get all of this advanced and honestly complex
...
```
Even tracing data is availble in the Super Graph web UI if tracing is enabled in the
config. By default it is for development.
![Query Tracing](/tracing.png "Super Graph Web UI Query Tracing")
## Authentication
You can only have one type of auth enabled. You can either pick Rails or JWT.
@ -610,6 +614,18 @@ database:
# even defaults.filter
filter: none
remotes:
- name: payments
id: stripe_id
url: http://rails_app:3000/stripe/$id
path: data
# pass_headers:
# - cookie
# - host
set_headers:
- name: Authorization
value: Bearer <stripe_api_key>
- # You can create new fields that have a
# real db table backing them
name: me

23
jsn/filter.go
View File

@ -8,7 +8,6 @@ import (
func Filter(w *bytes.Buffer, b []byte, keys []string) error {
var err error
kmap := make(map[uint64]struct{}, len(keys))
for i := range keys {
@ -39,8 +38,7 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
}
}
switch {
case state == expectKey:
if state == expectKey {
switch b[i] {
case '[':
if !isList {
@ -53,16 +51,19 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
} else {
_, err = w.Write([]byte("},{"))
}
item++
field = 0
case '"':
state = expectKeyClose
s = i
i++
item++
}
if err != nil {
return err
}
}
switch {
case state == expectKey && b[i] == '"':
state = expectKeyClose
s = i
case state == expectKeyClose && b[i] == '"':
state = expectColon
k = b[(s + 1):i]
@ -105,6 +106,12 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
e = i
case state == expectValue && b[i] == 'n':
state = expectNull
case state == expectNull && b[i] == 'l':
e = i
}
if e != 0 {

7
jsn/get.go
View File

@ -10,6 +10,7 @@ const (
expectColon
expectValue
expectString
expectNull
expectListClose
expectObjClose
expectBoolClose
@ -114,6 +115,12 @@ func Get(b []byte, keys [][]byte) []Field {
case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
e = i
case state == expectValue && b[i] == 'n':
state = expectNull
case state == expectNull && b[i] == 'l':
e = i
}
if e != 0 {

24
jsn/json_test.go
View File

@ -81,7 +81,8 @@ var (
"id": 11,
"full_name": "Arden Koss",
"email": "cristobalankunding@howewelch.org",
"__twitter_id": "2048666903444506956"
"__twitter_id": "2048666903444506956",
"something": null
},
{
"id": 12,
@ -106,6 +107,7 @@ var (
"full_name": "Sidney Stroman",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"something": null,
"embed": {
"id": 8,
"full_name": "Caroll Orn Sr.",
@ -137,7 +139,7 @@ var (
"__twitter_id": "2048666903444506956",
"embed": {
"id": 8,
"full_name": "Caroll Orn Sr.",
"full_name": null,
"email": "joannarau@hegmann.io",
"__twitter_id": "ABC123"
}
@ -216,7 +218,7 @@ func TestValue(t *testing.T) {
}
}
func TestFilter(t *testing.T) {
func TestFilter1(t *testing.T) {
var b bytes.Buffer
Filter(&b, []byte(input2), []string{"id", "full_name", "embed"})
@ -226,6 +228,20 @@ func TestFilter(t *testing.T) {
t.Error("Does not match expected json")
}
}
func TestFilter2(t *testing.T) {
value := `[{"id":1,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":100,"amount_refunded":0,"date":"01/01/2019","application":null,"billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}, {"id":2,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":0,"date":"02/18/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}},{"id":3,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":50,"date":"03/21/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}]`
var b bytes.Buffer
Filter(&b, []byte(value), []string{"id"})
expected := `[{"id":1},{"id":2},{"id":3}]`
if b.String() != expected {
t.Error("Does not match expected json")
}
}
func TestStrip(t *testing.T) {
path1 := [][]byte{[]byte("data"), []byte("users")}
value1 := Strip([]byte(input3), path1)
@ -266,7 +282,7 @@ func TestReplace(t *testing.T) {
"__twitter_id": "2048666903444506956",
"embed": {
"id": 8,
"full_name": "Caroll Orn Sr.",
"full_name": null,
"email": "joannarau@hegmann.io",
"some_list":[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]
}

6
jsn/replace.go
View File

@ -96,6 +96,12 @@ func Replace(w *bytes.Buffer, b []byte, from, to []Field) error {
case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
e = i
case state == expectValue && b[i] == 'n':
state = expectNull
case state == expectNull && b[i] == 'l':
e = i
}
if e != 0 {

6
jsn/strip.go
View File

@ -80,6 +80,12 @@ func Strip(b []byte, path [][]byte) []byte {
case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
e = i
case state == expectValue && b[i] == 'n':
state = expectNull
case state == expectNull && b[i] == 'l':
e = i
}
if e != 0 {

2
psql/psql.go
View File

@ -661,7 +661,7 @@ func renderOrderBy(w io.Writer, sel *qcode.Select) error {
case qcode.OrderDescNullsLast:
fmt.Fprintf(w, `%s_%d.ob.%s DESC NULLS LAST`, sel.Table, sel.ID, ob.Col)
default:
return fmt.Errorf("13: unexpected value %v (%t)", ob.Order, ob.Order)
return fmt.Errorf("13: unexpected value %v", ob.Order)
}
}
return nil

8
psql/psql_test.go
View File

@ -262,7 +262,7 @@ func oneToMany(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1.join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";`
sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1_join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {
@ -285,7 +285,7 @@ func belongsTo(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1.join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1_join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {
@ -308,7 +308,7 @@ func manyToMany(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1.join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1_join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {
@ -331,7 +331,7 @@ func manyToManyReverse(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1.join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";`
sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1_join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {

86
serv/core.go
View File

@ -92,24 +92,31 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
continue
}
st := time.Now()
b, err := r.Fn(req, id)
if err != nil {
return err
}
if conf.EnableTracing {
c.addTrace(s, st)
}
if len(r.Path) != 0 {
b = jsn.Strip(b, r.Path)
}
fils := []string{}
for i := range s.Cols {
fils = append(fils, s.Cols[i].Name)
}
var ob bytes.Buffer
if err = jsn.Filter(&ob, b, fils); err != nil {
return err
if len(s.Cols) != 0 {
err = jsn.Filter(&ob, b, colsToList(s.Cols))
if err != nil {
return err
}
} else {
ob.WriteString("null")
}
f := jsn.Field{[]byte(s.FieldName), ob.Bytes()}
@ -188,8 +195,8 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) (
return nil, 0, err
}
if conf.EnableTracing {
c.res.Extensions = &extensions{newTrace(st, time.Now(), qc)}
if conf.EnableTracing && len(qc.Query.Selects) != 0 {
c.addTrace(&qc.Query.Selects[0], st)
}
return []byte(root), skipped, nil
@ -200,6 +207,34 @@ func (c *coreContext) render(w io.Writer, data []byte) error {
return json.NewEncoder(w).Encode(c.res)
}
func (c *coreContext) addTrace(sel *qcode.Select, st time.Time) {
et := time.Now()
du := et.Sub(st)
if c.res.Extensions == nil {
c.res.Extensions = &extensions{&trace{
Version: 1,
StartTime: st,
Execution: execution{},
}}
}
c.res.Extensions.Tracing.EndTime = et
c.res.Extensions.Tracing.Duration = du
tr := resolver{
Path: []string{sel.Table},
ParentType: "Query",
FieldName: sel.Table,
ReturnType: "object",
StartOffset: 1,
Duration: du,
}
c.res.Extensions.Tracing.Execution.Resolvers =
append(c.res.Extensions.Tracing.Execution.Resolvers, tr)
}
func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) (
[][]byte,
map[uint64]*qcode.Select) {
@ -251,32 +286,11 @@ func authCheck(ctx *coreContext) bool {
return (ctx.Value(userIDKey) != nil)
}
func newTrace(st, et time.Time, qc *qcode.QCode) *trace {
if len(qc.Query.Selects) == 0 {
return nil
func colsToList(cols []qcode.Column) []string {
var f []string
for i := range cols {
f = append(f, cols[i].Name)
}
du := et.Sub(et)
sel := qc.Query.Selects[0]
t := &trace{
Version: 1,
StartTime: st,
EndTime: et,
Duration: du,
Execution: execution{
[]resolver{
resolver{
Path: []string{sel.Table},
ParentType: "Query",
FieldName: sel.Table,
ReturnType: "object",
StartOffset: 1,
Duration: du,
},
},
},
}
return t
return f
}