diff --git a/config/dev.yml b/config/dev.yml index 5261e24..2fe6357 100644 --- a/config/dev.yml +++ b/config/dev.yml @@ -110,14 +110,14 @@ database: remotes: - name: payments id: stripe_id - path: data - pass_headers: - - cookie - - host - # set_headers: - # - name: authorize - # value: Bearer 1234567890 url: http://rails_app:3000/stripe/$id + path: data + # pass_headers: + # - cookie + # - host + set_headers: + - name: Authorization + value: Bearer - # You can create new fields that have a # real db table backing them diff --git a/config/prod.yml b/config/prod.yml index 0e8ad2a..1ddcb9e 100644 --- a/config/prod.yml +++ b/config/prod.yml @@ -105,6 +105,18 @@ database: # even defaults.filter filter: none + # remotes: + # - name: payments + # id: stripe_id + # url: http://rails_app:3000/stripe/$id + # path: data + # # pass_headers: + # # - cookie + # # - host + # set_headers: + # - name: Authorization + # value: Bearer + - # You can create new fields that have a # real db table backing them name: me diff --git a/docs/.vuepress/public/tracing.png b/docs/.vuepress/public/tracing.png new file mode 100644 index 0000000..cfb268b Binary files /dev/null and b/docs/.vuepress/public/tracing.png differ diff --git a/docs/guide.md b/docs/guide.md index d671333..b496596 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -347,7 +347,7 @@ class AddSearchColumn < ActiveRecord::Migration[5.1] end ``` -## Stitching in REST APIs +## API Stitching It often happens that after fetching some data from the DB we need to call another API to fetch some more data and all this combined into a single JSON response. @@ -355,27 +355,26 @@ For example you need to list the last 3 payments made by a user. You will first Similiarly you might also have the need to fetch the users last tweet and include that too. Super Graph can handle this for you using it's `API Stitching` feature. -### API Stitching configuration +### Stripe API example The configuration is self explanatory. A `payments` field has been added under the `customers` table. This field is added to the `remotes` subsection that defines fields associated with `customers` that are remote and not real database columns. The `id` parameter maps a column from the `customers` table to the `$id` variable. In this case it maps `$id` to the `customer_id` column. ```yaml - tables: - - name: customers - - remotes: - - name: payments - id: customer_id - path: data - pass_headers: - - cookie - - host - # set_headers: - # - name: authorize - # value: Bearer 1234567890 - url: http://rails_app:3000/stripe/$id +tables: + - name: customers + remotes: + - name: payments + id: stripe_id + url: http://rails_app:3000/stripe/$id + path: data + # pass_headers: + # - cookie + # - host + set_headers: + - name: Authorization + value: Bearer ``` #### How do I make use of this? @@ -416,6 +415,11 @@ And voila here is the result. You get all of this advanced and honestly complex ... ``` +Even tracing data is availble in the Super Graph web UI if tracing is enabled in the +config. By default it is for development. + +![Query Tracing](/tracing.png "Super Graph Web UI Query Tracing") + ## Authentication You can only have one type of auth enabled. You can either pick Rails or JWT. @@ -610,6 +614,18 @@ database: # even defaults.filter filter: none + remotes: + - name: payments + id: stripe_id + url: http://rails_app:3000/stripe/$id + path: data + # pass_headers: + # - cookie + # - host + set_headers: + - name: Authorization + value: Bearer + - # You can create new fields that have a # real db table backing them name: me diff --git a/jsn/filter.go b/jsn/filter.go index 1a9f658..c55097a 100644 --- a/jsn/filter.go +++ b/jsn/filter.go @@ -8,7 +8,6 @@ import ( func Filter(w *bytes.Buffer, b []byte, keys []string) error { var err error - kmap := make(map[uint64]struct{}, len(keys)) for i := range keys { @@ -39,8 +38,7 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error { } } - switch { - case state == expectKey: + if state == expectKey { switch b[i] { case '[': if !isList { @@ -53,16 +51,19 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error { } else { _, err = w.Write([]byte("},{")) } - item++ field = 0 - case '"': - state = expectKeyClose - s = i - i++ + item++ } if err != nil { return err } + } + + switch { + case state == expectKey && b[i] == '"': + state = expectKeyClose + s = i + case state == expectKeyClose && b[i] == '"': state = expectColon k = b[(s + 1):i] @@ -105,6 +106,12 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error { case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'): e = i + + case state == expectValue && b[i] == 'n': + state = expectNull + + case state == expectNull && b[i] == 'l': + e = i } if e != 0 { diff --git a/jsn/get.go b/jsn/get.go index bc7d3a3..584ce84 100644 --- a/jsn/get.go +++ b/jsn/get.go @@ -10,6 +10,7 @@ const ( expectColon expectValue expectString + expectNull expectListClose expectObjClose expectBoolClose @@ -114,6 +115,12 @@ func Get(b []byte, keys [][]byte) []Field { case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'): e = i + + case state == expectValue && b[i] == 'n': + state = expectNull + + case state == expectNull && b[i] == 'l': + e = i } if e != 0 { diff --git a/jsn/json_test.go b/jsn/json_test.go index 1b939c2..ba545e6 100644 --- a/jsn/json_test.go +++ b/jsn/json_test.go @@ -81,7 +81,8 @@ var ( "id": 11, "full_name": "Arden Koss", "email": "cristobalankunding@howewelch.org", - "__twitter_id": "2048666903444506956" + "__twitter_id": "2048666903444506956", + "something": null }, { "id": 12, @@ -106,6 +107,7 @@ var ( "full_name": "Sidney Stroman", "email": "user0@demo.com", "__twitter_id": "2048666903444506956", + "something": null, "embed": { "id": 8, "full_name": "Caroll Orn Sr.", @@ -137,7 +139,7 @@ var ( "__twitter_id": "2048666903444506956", "embed": { "id": 8, - "full_name": "Caroll Orn Sr.", + "full_name": null, "email": "joannarau@hegmann.io", "__twitter_id": "ABC123" } @@ -216,7 +218,7 @@ func TestValue(t *testing.T) { } } -func TestFilter(t *testing.T) { +func TestFilter1(t *testing.T) { var b bytes.Buffer Filter(&b, []byte(input2), []string{"id", "full_name", "embed"}) @@ -226,6 +228,20 @@ func TestFilter(t *testing.T) { t.Error("Does not match expected json") } } + +func TestFilter2(t *testing.T) { + value := `[{"id":1,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":100,"amount_refunded":0,"date":"01/01/2019","application":null,"billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}, {"id":2,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":0,"date":"02/18/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}},{"id":3,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":50,"date":"03/21/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}]` + + var b bytes.Buffer + Filter(&b, []byte(value), []string{"id"}) + + expected := `[{"id":1},{"id":2},{"id":3}]` + + if b.String() != expected { + t.Error("Does not match expected json") + } +} + func TestStrip(t *testing.T) { path1 := [][]byte{[]byte("data"), []byte("users")} value1 := Strip([]byte(input3), path1) @@ -266,7 +282,7 @@ func TestReplace(t *testing.T) { "__twitter_id": "2048666903444506956", "embed": { "id": 8, - "full_name": "Caroll Orn Sr.", + "full_name": null, "email": "joannarau@hegmann.io", "some_list":[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}] } diff --git a/jsn/replace.go b/jsn/replace.go index a1216c3..43389a5 100644 --- a/jsn/replace.go +++ b/jsn/replace.go @@ -96,6 +96,12 @@ func Replace(w *bytes.Buffer, b []byte, from, to []Field) error { case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'): e = i + + case state == expectValue && b[i] == 'n': + state = expectNull + + case state == expectNull && b[i] == 'l': + e = i } if e != 0 { diff --git a/jsn/strip.go b/jsn/strip.go index 6427636..3b66fd0 100644 --- a/jsn/strip.go +++ b/jsn/strip.go @@ -80,6 +80,12 @@ func Strip(b []byte, path [][]byte) []byte { case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'): e = i + + case state == expectValue && b[i] == 'n': + state = expectNull + + case state == expectNull && b[i] == 'l': + e = i } if e != 0 { diff --git a/psql/psql.go b/psql/psql.go index 3748e04..211ac1d 100644 --- a/psql/psql.go +++ b/psql/psql.go @@ -661,7 +661,7 @@ func renderOrderBy(w io.Writer, sel *qcode.Select) error { case qcode.OrderDescNullsLast: fmt.Fprintf(w, `%s_%d.ob.%s DESC NULLS LAST`, sel.Table, sel.ID, ob.Col) default: - return fmt.Errorf("13: unexpected value %v (%t)", ob.Order, ob.Order) + return fmt.Errorf("13: unexpected value %v", ob.Order) } } return nil diff --git a/psql/psql_test.go b/psql/psql_test.go index b22fdb4..4c8d295 100644 --- a/psql/psql_test.go +++ b/psql/psql_test.go @@ -262,7 +262,7 @@ func oneToMany(t *testing.T) { } }` - sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1.join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";` + sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1_join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { @@ -285,7 +285,7 @@ func belongsTo(t *testing.T) { } }` - sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1.join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";` + sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1_join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { @@ -308,7 +308,7 @@ func manyToMany(t *testing.T) { } }` - sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1.join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";` + sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1_join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { @@ -331,7 +331,7 @@ func manyToManyReverse(t *testing.T) { } }` - sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1.join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";` + sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1_join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { diff --git a/serv/core.go b/serv/core.go index 6076cdb..e8e4f58 100644 --- a/serv/core.go +++ b/serv/core.go @@ -92,24 +92,31 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error { continue } + st := time.Now() + b, err := r.Fn(req, id) if err != nil { return err } + if conf.EnableTracing { + c.addTrace(s, st) + } + if len(r.Path) != 0 { b = jsn.Strip(b, r.Path) } - fils := []string{} - for i := range s.Cols { - fils = append(fils, s.Cols[i].Name) - } - var ob bytes.Buffer - if err = jsn.Filter(&ob, b, fils); err != nil { - return err + if len(s.Cols) != 0 { + err = jsn.Filter(&ob, b, colsToList(s.Cols)) + if err != nil { + return err + } + + } else { + ob.WriteString("null") } f := jsn.Field{[]byte(s.FieldName), ob.Bytes()} @@ -188,8 +195,8 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) ( return nil, 0, err } - if conf.EnableTracing { - c.res.Extensions = &extensions{newTrace(st, time.Now(), qc)} + if conf.EnableTracing && len(qc.Query.Selects) != 0 { + c.addTrace(&qc.Query.Selects[0], st) } return []byte(root), skipped, nil @@ -200,6 +207,34 @@ func (c *coreContext) render(w io.Writer, data []byte) error { return json.NewEncoder(w).Encode(c.res) } +func (c *coreContext) addTrace(sel *qcode.Select, st time.Time) { + et := time.Now() + du := et.Sub(st) + + if c.res.Extensions == nil { + c.res.Extensions = &extensions{&trace{ + Version: 1, + StartTime: st, + Execution: execution{}, + }} + } + + c.res.Extensions.Tracing.EndTime = et + c.res.Extensions.Tracing.Duration = du + + tr := resolver{ + Path: []string{sel.Table}, + ParentType: "Query", + FieldName: sel.Table, + ReturnType: "object", + StartOffset: 1, + Duration: du, + } + + c.res.Extensions.Tracing.Execution.Resolvers = + append(c.res.Extensions.Tracing.Execution.Resolvers, tr) +} + func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) ( [][]byte, map[uint64]*qcode.Select) { @@ -251,32 +286,11 @@ func authCheck(ctx *coreContext) bool { return (ctx.Value(userIDKey) != nil) } -func newTrace(st, et time.Time, qc *qcode.QCode) *trace { - if len(qc.Query.Selects) == 0 { - return nil +func colsToList(cols []qcode.Column) []string { + var f []string + + for i := range cols { + f = append(f, cols[i].Name) } - - du := et.Sub(et) - sel := qc.Query.Selects[0] - - t := &trace{ - Version: 1, - StartTime: st, - EndTime: et, - Duration: du, - Execution: execution{ - []resolver{ - resolver{ - Path: []string{sel.Table}, - ParentType: "Query", - FieldName: sel.Table, - ReturnType: "object", - StartOffset: 1, - Duration: du, - }, - }, - }, - } - - return t + return f }