From 7643e6ffa7256c33307554c93303aa0bfcbae398 Mon Sep 17 00:00:00 2001 From: Vikram Rangnekar Date: Thu, 6 Jun 2019 08:57:00 -0400 Subject: [PATCH] Add request / response debugging for remote joins --- config/dev.yml | 12 +++++--- demo | 11 ++++++- docker-compose.yml | 8 ++--- go.mod | 3 +- go.sum | 12 ++++++++ rails-app/config/environments/development.rb | 3 ++ rails-app/db/schema.rb | 10 +++--- serv/auth_rails.go | 18 +++++------ serv/config.go | 1 + serv/core.go | 6 ++-- serv/http.go | 3 +- serv/reso.go | 28 ++++++++++++++++- serv/serv.go | 32 +++++++++++--------- 13 files changed, 103 insertions(+), 44 deletions(-) diff --git a/config/dev.yml b/config/dev.yml index 07c417b..e5e464c 100644 --- a/config/dev.yml +++ b/config/dev.yml @@ -112,12 +112,14 @@ database: id: stripe_id url: http://rails_app:3000/stripe/$id path: data - # pass_headers: - # - cookie - # - host + # debug: true + pass_headers: + - cookie set_headers: - - name: Authorization - value: Bearer + - name: Host + value: 0.0.0.0 + # - name: Authorization + # value: Bearer - # You can create new fields that have a # real db table backing them diff --git a/demo b/demo index 43c20c1..0a386bc 100755 --- a/demo +++ b/demo @@ -1,12 +1,21 @@ #!/bin/bash if [ "$1" == "setup" ]; then + echo "Downloading pre-built docker images" + docker-compose -f rails-app/demo.yml pull + echo "Setting up the demo Rails app" docker-compose -f rails-app/demo.yml run web rake db:create db:migrate db:seed elif [ "$1" == "run" ]; then + echo "Deploying Super Graph and the demo Rails app" docker-compose -f rails-app/demo.yml up elif [ "$1" == "start" ]; then + echo "Downloading pre-built docker images" + docker-compose -f rails-app/demo.yml pull + echo "Setting up and deploying Super Graph and the demo Rails app" docker-compose -f rails-app/demo.yml run web rake db:create db:migrate db:seed docker-compose -f rails-app/demo.yml up +elif [ "$1" == "stop" ]; then + docker-compose -f rails-app/demo.yml down else - echo "./demo [setup|run|start]" + echo "./demo [setup|run|start|stop]" fi \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5900191..81f9290 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,14 +15,15 @@ services: target: go-build environment: GO_ENV: "development" - depends_on: - - db ports: - "8080:8080" volumes: - .:/app working_dir: /app command: fresh -c fresh.conf + depends_on: + - db + - rails_app rails_app: build: rails-app/. @@ -33,5 +34,4 @@ services: - "3000:3000" depends_on: - db - - super_graph - #- redis + # - redis diff --git a/go.mod b/go.mod index 1ea12b0..7cc0b6b 100644 --- a/go.mod +++ b/go.mod @@ -19,11 +19,12 @@ require ( github.com/mattn/go-isatty v0.0.7 // indirect github.com/onsi/ginkgo v1.8.0 // indirect github.com/onsi/gomega v1.5.0 // indirect + github.com/rs/zerolog v1.14.3 github.com/sirupsen/logrus v1.4.0 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/viper v1.3.1 github.com/valyala/fasttemplate v1.0.1 - golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 + golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 mellium.im/sasl v0.2.1 // indirect ) diff --git a/go.sum b/go.sum index 2f19776..38944cd 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/cespare/xxhash/v2 v2.0.0/go.mod h1:MaMeaVDXZNmTpkOyhVs3/WfjgobkbQgfrV github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,8 +59,12 @@ github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0= +github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg= github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -86,22 +91,29 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= diff --git a/rails-app/config/environments/development.rb b/rails-app/config/environments/development.rb index 7e6af8d..5945e84 100644 --- a/rails-app/config/environments/development.rb +++ b/rails-app/config/environments/development.rb @@ -60,4 +60,7 @@ Rails.application.configure do config.file_watcher = ActiveSupport::EventedFileUpdateChecker config.action_mailer.default_url_options = { host: 'localhost', port: 3000 } + + config.web_console.whitelisted_ips = ['10.0.0.0/8', '172.16.0.0/12', '192.168.0.0/16'] + end diff --git a/rails-app/db/schema.rb b/rails-app/db/schema.rb index 20b8165..7204423 100644 --- a/rails-app/db/schema.rb +++ b/rails-app/db/schema.rb @@ -2,11 +2,11 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `rails +# db:schema:load`. When creating a new database, `rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. diff --git a/serv/auth_rails.go b/serv/auth_rails.go index 8baf77f..33cf3b8 100644 --- a/serv/auth_rails.go +++ b/serv/auth_rails.go @@ -15,11 +15,11 @@ import ( func railsRedisHandler(next http.HandlerFunc) http.HandlerFunc { cookie := conf.Auth.Cookie if len(cookie) == 0 { - panic(errors.New("no auth.cookie defined")) + logger.Fatal().Msg("no auth.cookie defined") } if len(conf.Auth.Rails.URL) == 0 { - logger.Fatal(errors.New("no auth.rails.url defined")) + logger.Fatal().Msg("no auth.rails.url defined") } rp := &redis.Pool{ @@ -74,16 +74,16 @@ func railsRedisHandler(next http.HandlerFunc) http.HandlerFunc { func railsMemcacheHandler(next http.HandlerFunc) http.HandlerFunc { cookie := conf.Auth.Cookie if len(cookie) == 0 { - panic(errors.New("no auth.cookie defined")) + logger.Fatal().Msg("no auth.cookie defined") } if len(conf.Auth.Rails.URL) == 0 { - logger.Fatal(errors.New("no auth.rails.url defined")) + logger.Fatal().Msg("no auth.rails.url defined") } rURL, err := url.Parse(conf.Auth.Rails.URL) if err != nil { - logger.Fatal(err) + logger.Fatal().Err(err) } mc := memcache.New(rURL.Host) @@ -121,12 +121,12 @@ func railsMemcacheHandler(next http.HandlerFunc) http.HandlerFunc { func railsCookieHandler(next http.HandlerFunc) http.HandlerFunc { cookie := conf.Auth.Cookie if len(cookie) == 0 { - panic(errors.New("no auth.cookie defined")) + logger.Fatal().Msg("no auth.cookie defined") } ra, err := railsAuth(conf) if err != nil { - logger.Fatal(err) + logger.Fatal().Err(err) } return func(w http.ResponseWriter, r *http.Request) { @@ -137,14 +137,14 @@ func railsCookieHandler(next http.HandlerFunc) http.HandlerFunc { ck, err := r.Cookie(cookie) if err != nil { - logger.Error(err) + logger.Error().Err(err) next.ServeHTTP(w, r) return } userID, err := ra.ParseCookie(ck.Value) if err != nil { - logger.Error(err) + logger.Error().Err(err) next.ServeHTTP(w, r) return } diff --git a/serv/config.go b/serv/config.go index c425c57..3902ea1 100644 --- a/serv/config.go +++ b/serv/config.go @@ -76,6 +76,7 @@ type configRemote struct { ID string Path string URL string + Debug bool PassHeaders []string `mapstructure:"pass_headers"` SetHeaders []struct { Name string diff --git a/serv/core.go b/serv/core.go index 118acea..39c705f 100644 --- a/serv/core.go +++ b/serv/core.go @@ -100,7 +100,7 @@ func (c *coreContext) resolveRemote( // replacement data for the marked insertion points // key and value will be replaced by whats below toA := [1]jsn.Field{} - to := toA[:0] + to := toA[:1] // use the json key to find the related Select object k1 := xxhash.Sum64(field.Key) @@ -197,7 +197,7 @@ func (c *coreContext) resolveRemotes( return nil, nil } - go func(n int, s *qcode.Select) { + go func(n int, id []byte, s *qcode.Select) { defer wg.Done() st := time.Now() @@ -230,7 +230,7 @@ func (c *coreContext) resolveRemotes( } to[n] = jsn.Field{[]byte(s.FieldName), ob.Bytes()} - }(i, s) + }(i, id, s) } wg.Wait() diff --git a/serv/http.go b/serv/http.go index 3b3b88d..aa1e36d 100644 --- a/serv/http.go +++ b/serv/http.go @@ -10,6 +10,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/rs/zerolog/log" ) const ( @@ -137,7 +138,7 @@ func apiv1Http(w http.ResponseWriter, r *http.Request) { func errorResp(w http.ResponseWriter, err error) { if conf.DebugLevel > 0 { - logger.Error(err.Error()) + log.Error().Err(err) } w.WriteHeader(http.StatusBadRequest) diff --git a/serv/reso.go b/serv/reso.go index a3ac702..afa2663 100644 --- a/serv/reso.go +++ b/serv/reso.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "net/http" + "net/http/httputil" "strings" "github.com/cespare/xxhash/v2" @@ -100,7 +101,12 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { for _, v := range r.PassHeaders { h.Set(v, inReq.Header.Get(v)) } - req.Header = h + if len(h) != 0 { + if host, ok := h["Host"]; ok { + req.Host = host[0] + } + req.Header = h + } res, err := client.Do(req) if err != nil { @@ -108,6 +114,26 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { } defer res.Body.Close() + if r.Debug { + reqDump, err := httputil.DumpRequestOut(req, true) + if err != nil { + return nil, err + } + + resDump, err := httputil.DumpResponse(res, true) + if err != nil { + return nil, err + } + + logger.Warn().Msgf("Remote Request Debug:\n%s\n%s", + reqDump, resDump) + } + + if res.StatusCode != 200 { + return nil, + fmt.Errorf("server responded with a %d", res.StatusCode) + } + b, err := ioutil.ReadAll(res.Body) if err != nil { return nil, err diff --git a/serv/serv.go b/serv/serv.go index 0c76ced..f966987 100644 --- a/serv/serv.go +++ b/serv/serv.go @@ -15,7 +15,7 @@ import ( "github.com/dosco/super-graph/qcode" "github.com/go-pg/pg" "github.com/gobuffalo/flect" - "github.com/sirupsen/logrus" + "github.com/rs/zerolog" "github.com/spf13/viper" ) @@ -30,7 +30,7 @@ const ( ) var ( - logger *logrus.Logger + logger *zerolog.Logger conf *config db *pg.DB qcompile *qcode.Compiler @@ -38,15 +38,19 @@ var ( authFailBlock int ) -func initLog() *logrus.Logger { - log := logrus.New() - log.Formatter = new(logrus.TextFormatter) - log.Formatter.(*logrus.TextFormatter).DisableColors = false - log.Formatter.(*logrus.TextFormatter).DisableTimestamp = true - log.Level = logrus.TraceLevel - log.Out = os.Stdout +func initLog() *zerolog.Logger { + logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). + With().Caller().Logger() - return log + return &logger + /* + log := logrus.New() + logger.Formatter = new(logrus.TextFormatter) + logger.Formatter.(*logrus.TextFormatter).DisableColors = false + logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true + logger.Level = logrus.TraceLevel + logger.Out = os.Stdout + */ } func initConf() (*config, error) { @@ -178,17 +182,17 @@ func Init() { conf, err = initConf() if err != nil { - logger.Fatal(err) + logger.Fatal().Err(err) } db, err = initDB(conf) if err != nil { - logger.Fatal(err) + logger.Fatal().Err(err) } qcompile, pcompile, err = initCompilers(conf) if err != nil { - logger.Fatal(err) + logger.Fatal().Err(err) } initResolvers() @@ -219,7 +223,7 @@ func startHTTP() { srv.RegisterOnShutdown(func() { if err := db.Close(); err != nil { - logger.Println(err) + logger.Error().Err(err) } })