Merge branch 'master' into optimize-psql

config/dev.yml

@@ -112,12 +112,14 @@ database:
           id: stripe_id
           url: http://rails_app:3000/stripe/$id
           path: data
-          # pass_headers: 
+          # debug: true
-          #   - cookie
+          pass_headers: 
-          #   - host
+            - cookie
           set_headers:
-            - name: Authorization
+            - name: Host
-              value: Bearer <stripe_api_key>
+              value: 0.0.0.0
+            # - name: Authorization
+            #   value: Bearer <stripe_api_key>
 
     - # You can create new fields that have a
       # real db table backing them

demo

@@ -1,12 +1,21 @@
 #!/bin/bash
 
 if [ "$1" == "setup" ]; then
+  echo "Downloading pre-built docker images"
+  docker-compose -f rails-app/demo.yml pull
+  echo "Setting up the demo Rails app"
   docker-compose -f rails-app/demo.yml run web rake db:create db:migrate db:seed
 elif [ "$1" == "run" ]; then
+  echo "Deploying Super Graph and the demo Rails app"
   docker-compose -f rails-app/demo.yml up
 elif [ "$1" == "start" ]; then
+  echo "Downloading pre-built docker images"
+  docker-compose -f rails-app/demo.yml pull
+  echo "Setting up and deploying Super Graph and the demo Rails app"
   docker-compose -f rails-app/demo.yml run web rake db:create db:migrate db:seed
   docker-compose -f rails-app/demo.yml up
+elif [ "$1" == "stop" ]; then
+  docker-compose -f rails-app/demo.yml down
 else
-  echo "./demo [setup|run|start]"
+  echo "./demo [setup|run|start|stop]"
 fi

docker-compose.yml

@@ -15,14 +15,15 @@ services:
       target: go-build
     environment:
       GO_ENV: "development"
-    depends_on:
-      - db
     ports:
       - "8080:8080"
     volumes:
       - .:/app
     working_dir: /app
     command: fresh -c fresh.conf
+    depends_on:
+      - db
+      - rails_app
 
   rails_app:
     build: rails-app/.
@@ -33,5 +34,4 @@ services:
       - "3000:3000"
     depends_on:
       - db
-      - super_graph
+    # - redis
-      #- redis

go.mod

@@ -19,11 +19,12 @@ require (
 	github.com/mattn/go-isatty v0.0.7 // indirect
 	github.com/onsi/ginkgo v1.8.0 // indirect
 	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/rs/zerolog v1.14.3
 	github.com/sirupsen/logrus v1.4.0
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/viper v1.3.1
 	github.com/valyala/fasttemplate v1.0.1
-	golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 	golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223
 	mellium.im/sasl v0.2.1 // indirect
 )

go.sum

@@ -15,6 +15,7 @@ github.com/cespare/xxhash/v2 v2.0.0/go.mod h1:MaMeaVDXZNmTpkOyhVs3/WfjgobkbQgfrV
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -58,8 +59,12 @@ github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
+github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0=
+github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
 github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
@@ -86,22 +91,29 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=

rails-app/config/environments/development.rb

@@ -60,4 +60,7 @@ Rails.application.configure do
   config.file_watcher = ActiveSupport::EventedFileUpdateChecker
 
   config.action_mailer.default_url_options = { host: 'localhost', port: 3000 }
+  
+  config.web_console.whitelisted_ips = ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16']
+
 end

rails-app/db/schema.rb

@@ -2,11 +2,11 @@
 # of editing this file, please use the migrations feature of Active Record to
 # incrementally modify your database, and then regenerate this schema definition.
 #
-# Note that this schema.rb definition is the authoritative source for your
+# This file is the source Rails uses to define your schema when running `rails
-# database schema. If you need to create the application database on another
+# db:schema:load`. When creating a new database, `rails db:schema:load` tends to
-# system, you should be using db:schema:load, not running all the migrations
+# be faster and is potentially less error prone than running all of your
-# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# migrations from scratch. Old migrations may fail to apply correctly if those
-# you'll amass, the slower it'll run and the greater likelihood for issues).
+# migrations use external dependencies or application code.
 #
 # It's strongly recommended that you check this file into your version control system.
 

serv/auth_rails.go

@@ -15,11 +15,11 @@ import (
 func railsRedisHandler(next http.HandlerFunc) http.HandlerFunc {
 	cookie := conf.Auth.Cookie
 	if len(cookie) == 0 {
-		panic(errors.New("no auth.cookie defined"))
+		logger.Fatal().Msg("no auth.cookie defined")
 	}
 
 	if len(conf.Auth.Rails.URL) == 0 {
-		logger.Fatal(errors.New("no auth.rails.url defined"))
+		logger.Fatal().Msg("no auth.rails.url defined")
 	}
 
 	rp := &redis.Pool{
@@ -74,16 +74,16 @@ func railsRedisHandler(next http.HandlerFunc) http.HandlerFunc {
 func railsMemcacheHandler(next http.HandlerFunc) http.HandlerFunc {
 	cookie := conf.Auth.Cookie
 	if len(cookie) == 0 {
-		panic(errors.New("no auth.cookie defined"))
+		logger.Fatal().Msg("no auth.cookie defined")
 	}
 
 	if len(conf.Auth.Rails.URL) == 0 {
-		logger.Fatal(errors.New("no auth.rails.url defined"))
+		logger.Fatal().Msg("no auth.rails.url defined")
 	}
 
 	rURL, err := url.Parse(conf.Auth.Rails.URL)
 	if err != nil {
-		logger.Fatal(err)
+		logger.Fatal().Err(err)
 	}
 
 	mc := memcache.New(rURL.Host)
@@ -121,12 +121,12 @@ func railsMemcacheHandler(next http.HandlerFunc) http.HandlerFunc {
 func railsCookieHandler(next http.HandlerFunc) http.HandlerFunc {
 	cookie := conf.Auth.Cookie
 	if len(cookie) == 0 {
-		panic(errors.New("no auth.cookie defined"))
+		logger.Fatal().Msg("no auth.cookie defined")
 	}
 
 	ra, err := railsAuth(conf)
 	if err != nil {
-		logger.Fatal(err)
+		logger.Fatal().Err(err)
 	}
 
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -137,14 +137,14 @@ func railsCookieHandler(next http.HandlerFunc) http.HandlerFunc {
 
 		ck, err := r.Cookie(cookie)
 		if err != nil {
-			logger.Error(err)
+			logger.Error().Err(err)
 			next.ServeHTTP(w, r)
 			return
 		}
 
 		userID, err := ra.ParseCookie(ck.Value)
 		if err != nil {
-			logger.Error(err)
+			logger.Error().Err(err)
 			next.ServeHTTP(w, r)
 			return
 		}

serv/config.go

@@ -76,6 +76,7 @@ type configRemote struct {
 	ID          string
 	Path        string
 	URL         string
+	Debug       bool
 	PassHeaders []string `mapstructure:"pass_headers"`
 	SetHeaders  []struct {
 		Name  string

serv/core.go

@@ -100,7 +100,7 @@ func (c *coreContext) resolveRemote(
 	// replacement data for the marked insertion points
 	// key and value will be replaced by whats below
 	toA := [1]jsn.Field{}
-	to := toA[:0]
+	to := toA[:1]
 
 	// use the json key to find the related Select object
 	k1 := xxhash.Sum64(field.Key)
@@ -197,7 +197,7 @@ func (c *coreContext) resolveRemotes(
 			return nil, nil
 		}
 
-		go func(n int, s *qcode.Select) {
+		go func(n int, id []byte, s *qcode.Select) {
 			defer wg.Done()
 
 			st := time.Now()
@@ -230,7 +230,7 @@ func (c *coreContext) resolveRemotes(
 			}
 
 			to[n] = jsn.Field{[]byte(s.FieldName), ob.Bytes()}
-		}(i, s)
+		}(i, id, s)
 	}
 	wg.Wait()
 

serv/http.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog/log"
 )
 
 const (
@@ -137,7 +138,7 @@ func apiv1Http(w http.ResponseWriter, r *http.Request) {
 
 func errorResp(w http.ResponseWriter, err error) {
 	if conf.DebugLevel > 0 {
-		logger.Error(err.Error())
+		log.Error().Err(err)
 	}
 
 	w.WriteHeader(http.StatusBadRequest)

serv/reso.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"net/http/httputil"
 	"strings"
 
 	"github.com/cespare/xxhash/v2"
@@ -100,7 +101,12 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
 		for _, v := range r.PassHeaders {
 			h.Set(v, inReq.Header.Get(v))
 		}
-		req.Header = h
+		if len(h) != 0 {
+			if host, ok := h["Host"]; ok {
+				req.Host = host[0]
+			}
+			req.Header = h
+		}
 
 		res, err := client.Do(req)
 		if err != nil {
@@ -108,6 +114,26 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
 		}
 		defer res.Body.Close()
 
+		if r.Debug {
+			reqDump, err := httputil.DumpRequestOut(req, true)
+			if err != nil {
+				return nil, err
+			}
+
+			resDump, err := httputil.DumpResponse(res, true)
+			if err != nil {
+				return nil, err
+			}
+
+			logger.Warn().Msgf("Remote Request Debug:\n%s\n%s",
+				reqDump, resDump)
+		}
+
+		if res.StatusCode != 200 {
+			return nil,
+				fmt.Errorf("server responded with a %d", res.StatusCode)
+		}
+
 		b, err := ioutil.ReadAll(res.Body)
 		if err != nil {
 			return nil, err

serv/serv.go

@@ -15,7 +15,7 @@ import (
 	"github.com/dosco/super-graph/qcode"
 	"github.com/go-pg/pg"
 	"github.com/gobuffalo/flect"
-	"github.com/sirupsen/logrus"
+	"github.com/rs/zerolog"
 	"github.com/spf13/viper"
 )
 
@@ -30,7 +30,7 @@ const (
 )
 
 var (
-	logger        *logrus.Logger
+	logger        *zerolog.Logger
 	conf          *config
 	db            *pg.DB
 	qcompile      *qcode.Compiler
@@ -38,15 +38,19 @@ var (
 	authFailBlock int
 )
 
-func initLog() *logrus.Logger {
+func initLog() *zerolog.Logger {
-	log := logrus.New()
+	logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
-	log.Formatter = new(logrus.TextFormatter)
+		With().Caller().Logger()
-	log.Formatter.(*logrus.TextFormatter).DisableColors = false
-	log.Formatter.(*logrus.TextFormatter).DisableTimestamp = true
-	log.Level = logrus.TraceLevel
-	log.Out = os.Stdout
 
-	return log
+	return &logger
+	/*
+		log := logrus.New()
+		logger.Formatter = new(logrus.TextFormatter)
+		logger.Formatter.(*logrus.TextFormatter).DisableColors = false
+		logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true
+		logger.Level = logrus.TraceLevel
+		logger.Out = os.Stdout
+	*/
 }
 
 func initConf() (*config, error) {
@@ -178,17 +182,17 @@ func Init() {
 
 	conf, err = initConf()
 	if err != nil {
-		logger.Fatal(err)
+		logger.Fatal().Err(err)
 	}
 
 	db, err = initDB(conf)
 	if err != nil {
-		logger.Fatal(err)
+		logger.Fatal().Err(err)
 	}
 
 	qcompile, pcompile, err = initCompilers(conf)
 	if err != nil {
-		logger.Fatal(err)
+		logger.Fatal().Err(err)
 	}
 
 	initResolvers()
@@ -219,7 +223,7 @@ func startHTTP() {
 
 	srv.RegisterOnShutdown(func() {
 		if err := db.Close(); err != nil {
-			logger.Println(err)
+			logger.Error().Err(err)
 		}
 	})
 

slides/whygraphql.slide

@@ -0,0 +1,95 @@
+Why GraphQL
+GraphQL is the future of APIs
+Tags: GraphQL, API, GoLang, Postgres
+
+Vikram Rangnekar 
+https://twitter.com/@dosco
+
+* Trends, Why APIs are Important
+
+- An API first world
+- Rise of API only startups
+- Rise of integrations
+- Rise of fullstack developers
+- Rise of Single-Page-Apps
+
+* Web Development Today
+
+A big part of web development is building CRUD APIs to read
+update and delete things from a database.
+
+Get all products that belong to user 5 
+.link http://your-startup.com/apis/v1/users/5/products
+
+Another way to do this
+.link http://your-startup.com/apis/v1/products?user_id=5
+
+Maybe you just need only cheaper products
+.link http://your-startup.com/apis/v1/products?user_id=5&price_under=12
+
+* So what's the problem here?
+
+- Too many decisions no real standards
+- Harder to be consistant
+- Rinse and repeat for every new API
+- Too much data over the wire
+- Boring, we rather work on more interesting things
+- Slows down dev. cycles
+
+* GraphQL
+
+  query {
+    user(id: 5) {
+
+      products {
+        id
+        name
+        photo : image
+
+        customers {
+          name
+          email
+        }
+      }
+    }
+  }
+
+* Perceived bad parts of GraphQL
+
+- Perceived as new and shiny
+- Not cache friendly (Not true)
+- Only best data fetching 
+- New frameworks to learn
+- Is it just an API gateway?
+- Write more code
+
+* Super Graph - An instant GraphQL API for Postgres
+
+- GraphQL without writing any code
+- Automatically learns your database
+- Full text search, Aggregations, etc
+- Supports Rails cookies and JWT tokens 
+- Join with remote REST APIs
+- Highly optimized and fast Postgres SQL queries
+- High performance GO codebase
+- Tiny docker image and low memory requirements
+
+* Let's talk about Postgres DB
+
+- Atomicity, Consistency, Isolation, and Durability
+- Full support for JSON
+- Full-text search
+- Graph DB (WITH RESURSIVE clause)
+- Timeseries DB (WINDOW functions)
+- GIS Location DB (PostGIS)
+- Custom Columns (Bloomfilter, etc)
+- Nearest Neighbour Searches (GIST Index)
+- Row level security
+- Versioning support
+
+  All your micro-services in one
+
+* Super Graph gives Fullstack Devs superpowers
+
+
+.iframe https://giphy.com/gifs/3o6ZsYzuLyRfSGX4f6/html5 500 900