Add tracing for API stitching

config/dev.yml

@@ -110,14 +110,14 @@ database:
       remotes:
         - name: payments
           id: stripe_id
-          path: data
-          pass_headers: 
-            - cookie
-            - host
-          # set_headers:
-          #   - name: authorize
-          #     value: Bearer 1234567890
           url: http://rails_app:3000/stripe/$id
+          path: data
+          # pass_headers: 
+          #   - cookie
+          #   - host
+          set_headers:
+            - name: Authorization
+              value: Bearer <stripe_api_key>
 
     - # You can create new fields that have a
       # real db table backing them

config/prod.yml

@@ -105,6 +105,18 @@ database:
       # even defaults.filter
       filter: none
 
+      # remotes:
+      #   - name: payments
+      #     id: stripe_id
+      #     url: http://rails_app:3000/stripe/$id
+      #     path: data
+      #     # pass_headers: 
+      #     #   - cookie
+      #     #   - host
+      #     set_headers:
+      #       - name: Authorization
+      #         value: Bearer <stripe_api_key>
+
     - # You can create new fields that have a
       # real db table backing them
       name: me

docs/guide.md

@@ -347,7 +347,7 @@ class AddSearchColumn < ActiveRecord::Migration[5.1]
 end
 ```
 
-## Stitching in REST APIs
+## API Stitching
 
 It often happens that after fetching some data from the DB we need to call another API to fetch some more data and all this combined into a single JSON response.
 
@@ -355,27 +355,26 @@ For example you need to list the last 3 payments made by a user. You will first
 
 Similiarly you might also have the need to fetch the users last tweet and include that too. Super Graph can handle this for you using it's `API Stitching` feature.
 
-### API Stitching configuration
+### Stripe API example
 
 The configuration is self explanatory. A `payments` field has been added under the `customers` table. This field is added to the `remotes` subsection that defines fields associated with `customers` that are remote and not real database columns.
 
 The `id` parameter maps a column from the `customers` table to the `$id` variable. In this case it maps `$id` to the `customer_id` column.
 
 ```yaml
- tables:
+tables:
-    - name: customers
+  - name: customers
-
+    remotes:
-      remotes:
+      - name: payments
-        - name: payments
+        id: stripe_id
-          id: customer_id
+        url: http://rails_app:3000/stripe/$id
-          path: data
+        path: data
-          pass_headers: 
+        # pass_headers: 
-            - cookie
+        #   - cookie
-            - host
+        #   - host
-          # set_headers:
+        set_headers:
-          #   - name: authorize
+          - name: Authorization
-          #     value: Bearer 1234567890
+            value: Bearer <stripe_api_key>
-          url: http://rails_app:3000/stripe/$id
 ```
 
 #### How do I make use of this?
@@ -416,6 +415,11 @@ And voila here is the result. You get all of this advanced and honestly complex
       ...
 ```
 
+Even tracing data is availble in the Super Graph web UI if tracing is enabled in the
+config. By default it is for development.
+
+![Query Tracing](/tracing.png "Super Graph Web UI Query Tracing")
+
 ## Authentication
 
 You can only have one type of auth enabled. You can either pick Rails or JWT. 
@@ -610,6 +614,18 @@ database:
       # even defaults.filter
       filter: none
 
+      remotes:
+        - name: payments
+          id: stripe_id
+          url: http://rails_app:3000/stripe/$id
+          path: data
+          # pass_headers: 
+          #   - cookie
+          #   - host
+          set_headers:
+            - name: Authorization
+              value: Bearer <stripe_api_key>
+
     - # You can create new fields that have a
       # real db table backing them
       name: me

jsn/filter.go

@@ -8,7 +8,6 @@ import (
 
 func Filter(w *bytes.Buffer, b []byte, keys []string) error {
 	var err error
-
 	kmap := make(map[uint64]struct{}, len(keys))
 
 	for i := range keys {
@@ -39,8 +38,7 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
 			}
 		}
 
-		switch {
+		if state == expectKey {
-		case state == expectKey:
 			switch b[i] {
 			case '[':
 				if !isList {
@@ -53,16 +51,19 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
 				} else {
 					_, err = w.Write([]byte("},{"))
 				}
-				item++
 				field = 0
-			case '"':
+				item++
-				state = expectKeyClose
-				s = i
-				i++
 			}
 			if err != nil {
 				return err
 			}
+		}
+
+		switch {
+		case state == expectKey && b[i] == '"':
+			state = expectKeyClose
+			s = i
+
 		case state == expectKeyClose && b[i] == '"':
 			state = expectColon
 			k = b[(s + 1):i]
@@ -105,6 +106,12 @@ func Filter(w *bytes.Buffer, b []byte, keys []string) error {
 
 		case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
 			e = i
+
+		case state == expectValue && b[i] == 'n':
+			state = expectNull
+
+		case state == expectNull && b[i] == 'l':
+			e = i
 		}
 
 		if e != 0 {

jsn/get.go

@@ -10,6 +10,7 @@ const (
 	expectColon
 	expectValue
 	expectString
+	expectNull
 	expectListClose
 	expectObjClose
 	expectBoolClose
@@ -114,6 +115,12 @@ func Get(b []byte, keys [][]byte) []Field {
 
 		case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
 			e = i
+
+		case state == expectValue && b[i] == 'n':
+			state = expectNull
+
+		case state == expectNull && b[i] == 'l':
+			e = i
 		}
 
 		if e != 0 {

jsn/json_test.go

@@ -81,7 +81,8 @@ var (
 			"id": 11,
 			"full_name": "Arden Koss",
 			"email": "cristobalankunding@howewelch.org",
-			"__twitter_id": "2048666903444506956"
+			"__twitter_id": "2048666903444506956",
+			"something": null
 		},
 		{
 			"id": 12,
@@ -106,6 +107,7 @@ var (
 		"full_name": "Sidney Stroman",
 		"email": "user0@demo.com",
 		"__twitter_id": "2048666903444506956",
+		"something": null,
 		"embed": {
 			"id": 8,
 			"full_name": "Caroll Orn Sr.",
@@ -137,7 +139,7 @@ var (
 		"__twitter_id": "2048666903444506956",
 		"embed": {
 			"id": 8,
-			"full_name": "Caroll Orn Sr.",
+			"full_name": null,
 			"email": "joannarau@hegmann.io",
 			"__twitter_id": "ABC123"
 		}
@@ -216,7 +218,7 @@ func TestValue(t *testing.T) {
 	}
 }
 
-func TestFilter(t *testing.T) {
+func TestFilter1(t *testing.T) {
 	var b bytes.Buffer
 	Filter(&b, []byte(input2), []string{"id", "full_name", "embed"})
 
@@ -226,6 +228,20 @@ func TestFilter(t *testing.T) {
 		t.Error("Does not match expected json")
 	}
 }
+
+func TestFilter2(t *testing.T) {
+	value := `[{"id":1,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":100,"amount_refunded":0,"date":"01/01/2019","application":null,"billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}},   {"id":2,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":0,"date":"02/18/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}},{"id":3,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":50,"date":"03/21/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}]`
+
+	var b bytes.Buffer
+	Filter(&b, []byte(value), []string{"id"})
+
+	expected := `[{"id":1},{"id":2},{"id":3}]`
+
+	if b.String() != expected {
+		t.Error("Does not match expected json")
+	}
+}
+
 func TestStrip(t *testing.T) {
 	path1 := [][]byte{[]byte("data"), []byte("users")}
 	value1 := Strip([]byte(input3), path1)
@@ -266,7 +282,7 @@ func TestReplace(t *testing.T) {
 		"__twitter_id": "2048666903444506956",
 		"embed": {
 			"id": 8,
-			"full_name": "Caroll Orn Sr.",
+			"full_name": null,
 			"email": "joannarau@hegmann.io",
 			"some_list":[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]
 		}

jsn/replace.go

@@ -96,6 +96,12 @@ func Replace(w *bytes.Buffer, b []byte, from, to []Field) error {
 
 		case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
 			e = i
+
+		case state == expectValue && b[i] == 'n':
+			state = expectNull
+
+		case state == expectNull && b[i] == 'l':
+			e = i
 		}
 
 		if e != 0 {

jsn/strip.go

@@ -80,6 +80,12 @@ func Strip(b []byte, path [][]byte) []byte {
 
 		case state == expectBoolClose && (b[i] == 'e' || b[i] == 'E'):
 			e = i
+
+		case state == expectValue && b[i] == 'n':
+			state = expectNull
+
+		case state == expectNull && b[i] == 'l':
+			e = i
 		}
 
 		if e != 0 {

psql/psql.go

@@ -661,7 +661,7 @@ func renderOrderBy(w io.Writer, sel *qcode.Select) error {
 		case qcode.OrderDescNullsLast:
 			fmt.Fprintf(w, `%s_%d.ob.%s DESC NULLS LAST`, sel.Table, sel.ID, ob.Col)
 		default:
-			return fmt.Errorf("13: unexpected value %v (%t)", ob.Order, ob.Order)
+			return fmt.Errorf("13: unexpected value %v", ob.Order)
 		}
 	}
 	return nil

psql/psql_test.go

@@ -262,7 +262,7 @@ func oneToMany(t *testing.T) {
 		}
 	}`
 
-	sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1.join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";`
+	sql := `SELECT json_object_agg('users', users) FROM (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "users_0"."email" AS "email", "products_1_join"."products" AS "products") AS "sel_0")) AS "users" FROM (SELECT "users"."email", "users"."id" FROM "users" WHERE ((("users"."id") = ('{{user_id}}'))) LIMIT ('20') :: integer) AS "users_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name", "products_1"."price" AS "price") AS "sel_1")) AS "products" FROM (SELECT "products"."name", "products"."price" FROM "products" WHERE ((("products"."user_id") = ("users_0"."id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "users_0") AS "done_1337";`
 
 	resSQL, err := compileGQLToPSQL(gql)
 	if err != nil {
@@ -285,7 +285,7 @@ func belongsTo(t *testing.T) {
 		}
 	}`
 
-	sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1.join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
+	sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "products_0"."price" AS "price", "users_1_join"."users" AS "users") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."price", "products"."user_id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("users"), '[]') AS "users" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "users_1"."email" AS "email") AS "sel_1")) AS "users" FROM (SELECT "users"."email" FROM "users" WHERE ((("users"."id") = ("products_0"."user_id"))) LIMIT ('20') :: integer) AS "users_1" LIMIT ('20') :: integer) AS "users_1") AS "users_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
 
 	resSQL, err := compileGQLToPSQL(gql)
 	if err != nil {
@@ -308,7 +308,7 @@ func manyToMany(t *testing.T) {
 		}
 	}`
 
-	sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1.join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1.join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
+	sql := `SELECT json_object_agg('products', products) FROM (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."name" AS "name", "customers_1_join"."customers" AS "customers") AS "sel_0")) AS "products" FROM (SELECT "products"."name", "products"."id" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8))) LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "customers_1"."email" AS "email", "customers_1"."full_name" AS "full_name") AS "sel_1")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name" FROM "customers" LEFT OUTER JOIN "purchases" ON (("purchases"."product_id") = ("products_0"."id")) WHERE ((("customers"."id") = ("purchases"."customer_id"))) LIMIT ('20') :: integer) AS "customers_1" LIMIT ('20') :: integer) AS "customers_1") AS "customers_1_join" ON ('true') LIMIT ('20') :: integer) AS "products_0") AS "done_1337";`
 
 	resSQL, err := compileGQLToPSQL(gql)
 	if err != nil {
@@ -331,7 +331,7 @@ func manyToManyReverse(t *testing.T) {
 		}
 	}`
 
-	sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1.join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1.join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";`
+	sql := `SELECT json_object_agg('customers', customers) FROM (SELECT coalesce(json_agg("customers"), '[]') AS "customers" FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "customers_0"."email" AS "email", "customers_0"."full_name" AS "full_name", "products_1_join"."products" AS "products") AS "sel_0")) AS "customers" FROM (SELECT "customers"."email", "customers"."full_name", "customers"."id" FROM "customers" LIMIT ('20') :: integer) AS "customers_0" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("products"), '[]') AS "products" FROM (SELECT row_to_json((SELECT "sel_1" FROM (SELECT "products_1"."name" AS "name") AS "sel_1")) AS "products" FROM (SELECT "products"."name" FROM "products" LEFT OUTER JOIN "purchases" ON (("purchases"."customer_id") = ("customers_0"."id")) WHERE ((("products"."id") = ("purchases"."product_id"))) LIMIT ('20') :: integer) AS "products_1" LIMIT ('20') :: integer) AS "products_1") AS "products_1_join" ON ('true') LIMIT ('20') :: integer) AS "customers_0") AS "done_1337";`
 
 	resSQL, err := compileGQLToPSQL(gql)
 	if err != nil {

serv/core.go

@@ -92,24 +92,31 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
 			continue
 		}
 
+		st := time.Now()
+
 		b, err := r.Fn(req, id)
 		if err != nil {
 			return err
 		}
 
+		if conf.EnableTracing {
+			c.addTrace(s, st)
+		}
+
 		if len(r.Path) != 0 {
 			b = jsn.Strip(b, r.Path)
 		}
 
-		fils := []string{}
-		for i := range s.Cols {
-			fils = append(fils, s.Cols[i].Name)
-		}
-
 		var ob bytes.Buffer
 
-		if err = jsn.Filter(&ob, b, fils); err != nil {
+		if len(s.Cols) != 0 {
-			return err
+			err = jsn.Filter(&ob, b, colsToList(s.Cols))
+			if err != nil {
+				return err
+			}
+
+		} else {
+			ob.WriteString("null")
 		}
 
 		f := jsn.Field{[]byte(s.FieldName), ob.Bytes()}
@@ -188,8 +195,8 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode, vars variables) (
 		return nil, 0, err
 	}
 
-	if conf.EnableTracing {
+	if conf.EnableTracing && len(qc.Query.Selects) != 0 {
-		c.res.Extensions = &extensions{newTrace(st, time.Now(), qc)}
+		c.addTrace(&qc.Query.Selects[0], st)
 	}
 
 	return []byte(root), skipped, nil
@@ -200,6 +207,34 @@ func (c *coreContext) render(w io.Writer, data []byte) error {
 	return json.NewEncoder(w).Encode(c.res)
 }
 
+func (c *coreContext) addTrace(sel *qcode.Select, st time.Time) {
+	et := time.Now()
+	du := et.Sub(st)
+
+	if c.res.Extensions == nil {
+		c.res.Extensions = &extensions{&trace{
+			Version:   1,
+			StartTime: st,
+			Execution: execution{},
+		}}
+	}
+
+	c.res.Extensions.Tracing.EndTime = et
+	c.res.Extensions.Tracing.Duration = du
+
+	tr := resolver{
+		Path:        []string{sel.Table},
+		ParentType:  "Query",
+		FieldName:   sel.Table,
+		ReturnType:  "object",
+		StartOffset: 1,
+		Duration:    du,
+	}
+
+	c.res.Extensions.Tracing.Execution.Resolvers =
+		append(c.res.Extensions.Tracing.Execution.Resolvers, tr)
+}
+
 func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) (
 	[][]byte,
 	map[uint64]*qcode.Select) {
@@ -251,32 +286,11 @@ func authCheck(ctx *coreContext) bool {
 	return (ctx.Value(userIDKey) != nil)
 }
 
-func newTrace(st, et time.Time, qc *qcode.QCode) *trace {
+func colsToList(cols []qcode.Column) []string {
-	if len(qc.Query.Selects) == 0 {
+	var f []string
-		return nil
+
+	for i := range cols {
+		f = append(f, cols[i].Name)
 	}
-
+	return f
-	du := et.Sub(et)
-	sel := qc.Query.Selects[0]
-
-	t := &trace{
-		Version:   1,
-		StartTime: st,
-		EndTime:   et,
-		Duration:  du,
-		Execution: execution{
-			[]resolver{
-				resolver{
-					Path:        []string{sel.Table},
-					ParentType:  "Query",
-					FieldName:   sel.Table,
-					ReturnType:  "object",
-					StartOffset: 1,
-					Duration:    du,
-				},
-			},
-		},
-	}
-
-	return t
 }