Fix for encrypted cursor in production mode bug

Dockerfile

@@ -6,14 +6,19 @@ RUN yarn
 RUN yarn build
 
 # stage: 2
-FROM golang:1.13.4-alpine as go-build
+FROM golang:1.14-alpine as go-build
 RUN apk update && \
     apk add --no-cache make && \
     apk add --no-cache git && \
+    apk add --no-cache jq && \
     apk add --no-cache upx=3.95-r2
 
 RUN GO111MODULE=off go get -u github.com/rafaelsq/wtc
 
+ARG SOPS_VERSION=3.5.0
+ADD https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux /usr/local/bin/sops
+RUN chmod 755 /usr/local/bin/sops
+
 WORKDIR /app
 COPY . /app
 
@@ -36,10 +41,15 @@ RUN mkdir -p /config
 COPY --from=go-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=go-build /app/config/* /config/
 COPY --from=go-build /app/super-graph .
+COPY --from=go-build /app/scripts/start.sh .
+COPY --from=go-build /usr/local/bin/sops .
 
 RUN chmod +x /super-graph
+RUN chmod +x /start.sh
+
 USER nobody
 
-EXPOSE 8080
+ENV GO_ENV production
 
-CMD ./super-graph serv
+ENTRYPOINT ["./start.sh"]
+CMD ["./super-graph", "serv"]

config/dev.yml

@@ -36,6 +36,15 @@ migrations_path: ./config/migrations
 # encrypting the cursor data
 secret_key: supercalifajalistics
 
+# CORS: A list of origins a cross-domain request can be executed from. 
+# If the special * value is present in the list, all origins will be allowed. 
+# An origin may contain a wildcard (*) to replace 0 or more 
+# characters (i.e.: http://*.domain.com).
+cors_allowed_origins: ["*"]
+
+# Debug Cross Origin Resource Sharing requests
+cors_debug: true
+
 # Postgres related environment Variables
 # SG_DATABASE_HOST
 # SG_DATABASE_PORT

docs/guide.md

@@ -292,6 +292,12 @@ for (i = 0; i < 10; i++) {
 }
 ```
 
+If you want to import a lot of data using a CSV file is the best and fastest option. The `import_csv` command uses the `COPY FROM` Postgres method to load massive amounts of data into tables. The first line of the CSV file must be the header with column names.
+
+```javascript
+var post_count = import_csv("posts", "posts.csv")
+```
+
 You can generate the following fake data for your seeding purposes. Below is the list of fake data functions supported by the built-in fake data library. For example `fake.image_url()` will generate a fake image url or `fake.shuffle_strings(['hello', 'world', 'cool'])` will generate a randomly shuffled version of that array of strings or `fake.rand_string(['hello', 'world', 'cool'])` will return a random string from the array provided.
 
 ```
@@ -1975,7 +1981,7 @@ To use Yugabyte in your local development flow just uncomment the following line
 
 ## Developing Super Graph
 
-If you want to build and run Super Graph from code then the below commands will build the web ui and launch Super Graph in developer mode with a watcher to rebuild on code changes. And the demo rails app is also launched to make it essier to test changes.
+If you want to build and run Super Graph from code then the below commands will build the web ui and launch Super Graph in developer mode with a watcher to rebuild on code changes. And the demo rails app is also launched to make it easier to test changes.
 
 ```bash
 

go.mod

@@ -2,7 +2,6 @@ module github.com/dosco/super-graph
 
 require (
 	github.com/GeertJohan/go.rice v1.0.0
-	github.com/Masterminds/semver v1.5.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
@@ -12,7 +11,6 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/dlclark/regexp2 v1.2.0 // indirect
 	github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733
-	github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/garyburd/redigo v1.6.0
 	github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
@@ -23,6 +21,7 @@ require (
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/pelletier/go-toml v1.4.0 // indirect
 	github.com/pkg/errors v0.8.1
+	github.com/rs/cors v1.7.0
 	github.com/rs/zerolog v1.15.0
 	github.com/spf13/afero v1.2.2 // indirect
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -5,8 +5,6 @@ github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
-github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
-github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -54,8 +52,6 @@ github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 h1:cyNc40Dx5YNEO94idePU8rhVd3dn+sd04Arh0kDBAaw=
 github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
-github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c h1:/bXaeEuNG6V0HeyEGw11DYLW5BGsOPlcVRIXbHNUWSo=
-github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc=
@@ -184,6 +180,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
+github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0 h1:uPRuwkWF4J6fGsJ2R0Gn2jB1EQiav9k3S6CSdygQJXY=

psql/query.go

@@ -17,6 +17,10 @@ const (
 	closeBlock = 500
 )
 
+var (
+	ErrAllTablesSkipped = errors.New("all tables skipped. cannot render query")
+)
+
 type Variables map[string]json.RawMessage
 
 type Config struct {
@@ -107,7 +111,7 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w io.Writer, vars Variables) (
 	io.WriteString(c.w, `) as "__root" FROM `)
 
 	if i == 0 {
-		return 0, errors.New("all tables skipped. cannot render query")
+		return 0, ErrAllTablesSkipped
 	}
 
 	var ignored uint32
@@ -1127,9 +1131,15 @@ func (c *compilerContext) renderVal(ex *qcode.Exp, vars map[string]string, col *
 
 	switch ex.Type {
 	case qcode.ValVar:
-		if val, ok := vars[ex.Val]; ok {
+		val, ok := vars[ex.Val]
+		switch {
+		case ok && strings.HasPrefix(val, "sql:"):
+			io.WriteString(c.w, ` (`)
+			io.WriteString(c.w, val[4:])
+			io.WriteString(c.w, `)`)
+		case ok:
 			squoted(c.w, val)
-		} else {
+		default:
 			io.WriteString(c.w, ` '{{`)
 			io.WriteString(c.w, ex.Val)
 			io.WriteString(c.w, `}}'`)

rails/auth.go

@@ -4,8 +4,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
-	"github.com/Masterminds/semver"
 	"github.com/adjust/gorails/marshal"
 )
 
@@ -37,17 +38,20 @@ func NewAuth(version, secret string) (*Auth, error) {
 		AuthSalt: authSalt,
 	}
 
-	ver, err := semver.NewVersion(version)
-	if err != nil {
-		return nil, fmt.Errorf("rails auth: %s", err)
+	var v1, v2 int
+	var err error
+
+	sv := strings.Split(version, ".")
+	if len(sv) >= 2 {
+		if v1, err = strconv.Atoi(sv[0]); err != nil {
+			return nil, err
+		}
+		if v2, err = strconv.Atoi(sv[1]); err != nil {
+			return nil, err
+		}
 	}
 
-	gt52, err := semver.NewConstraint(">= 5.2")
-	if err != nil {
-		return nil, fmt.Errorf("rails auth: %s", err)
-	}
-
-	if gt52.Check(ver) {
+	if v1 >= 5 && v2 >= 2 {
 		ra.Cipher = railsCipher52
 	} else {
 		ra.Cipher = railsCipher

scripts/start.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+if [ $1 = "secrets" ]
+then
+  ./sops --config ./config "${@:2}"
+  exit 0
+fi
+
+if test -f "./config/$SECRETS_FILE"
+then
+  ./sops --config ./config exec-env "./config/$SECRETS_FILE" "$*"
+else
+  $@
+fi

serv/args.go

@@ -57,8 +57,6 @@ func argMap(ctx context.Context, vars []byte) func(w io.Writer, tag string) (int
 			return w.Write(v1)
 		}
 
-		fmt.Println(">>>", tag, string(v))
-
 		return w.Write(escQuote(fields[0].Value))
 	}
 }

serv/cmd_seed.go

@@ -3,6 +3,7 @@ package serv
 import (
 	"bytes"
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,9 +11,12 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strconv"
+	"strings"
 
 	"github.com/brianvoe/gofakeit"
 	"github.com/dop251/goja"
+	"github.com/jackc/pgx/v4"
 	"github.com/spf13/cobra"
 	"github.com/valyala/fasttemplate"
 )
@@ -42,6 +46,7 @@ func cmdDBSeed(cmd *cobra.Command, args []string) {
 
 	vm := goja.New()
 	vm.Set("graphql", graphQLFunc)
+	vm.Set("import_csv", importCSV)
 
 	console := vm.NewObject()
 	console.Set("log", logFunc) //nolint: errcheck
@@ -129,6 +134,106 @@ func graphQLFunc(query string, data interface{}, opt map[string]string) map[stri
 	return val
 }
 
+type csvSource struct {
+	rows [][]string
+	i    int
+}
+
+func NewCSVSource(filename string) (*csvSource, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r := csv.NewReader(f)
+	rows, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	return &csvSource{rows: rows}, nil
+}
+
+func (c *csvSource) Next() bool {
+	return c.i < len(c.rows)
+}
+
+func (c *csvSource) Values() ([]interface{}, error) {
+	var vals []interface{}
+	var err error
+
+	for _, v := range c.rows[c.i] {
+		switch {
+		case len(v) == 0:
+			vals = append(vals, "")
+		case isDigit(v):
+			var n int
+			if n, err = strconv.Atoi(v); err == nil {
+				vals = append(vals, n)
+			}
+		case strings.EqualFold(v, "true") || strings.EqualFold(v, "false"):
+			var b bool
+			if b, err = strconv.ParseBool(v); err == nil {
+				vals = append(vals, b)
+			}
+		default:
+			vals = append(vals, v)
+		}
+
+		if err != nil {
+			return nil, fmt.Errorf("%w (line no %d)", err, c.i)
+		}
+	}
+	c.i++
+
+	return vals, nil
+}
+
+func isDigit(v string) bool {
+	for i := range v {
+		if v[i] < '0' || v[i] > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *csvSource) Err() error {
+	return nil
+}
+
+func importCSV(table, filename string) int64 {
+	if filename[0] != '/' {
+		filename = path.Join(confPath, filename)
+	}
+
+	s, err := NewCSVSource(filename)
+	if err != nil {
+		errlog.Fatal().Err(err).Send()
+	}
+
+	var cols []string
+	colval, _ := s.Values()
+
+	for _, c := range colval {
+		cols = append(cols, c.(string))
+	}
+
+	n, err := db.CopyFrom(
+		context.Background(),
+		pgx.Identifier{table},
+		cols,
+		s)
+
+	if err != nil {
+		err = fmt.Errorf("%w (line no %d)", err, s.i)
+		errlog.Fatal().Err(err).Send()
+	}
+
+	return n
+}
+
 //nolint: errcheck
 func logFunc(args ...interface{}) {
 	for _, arg := range args {

serv/config.go

@@ -26,11 +26,13 @@ type config struct {
 	EnableTracing  bool   `mapstructure:"enable_tracing"`
 	UseAllowList   bool   `mapstructure:"use_allow_list"`
 	Production     bool
-	WatchAndReload bool   `mapstructure:"reload_on_config_change"`
-	AuthFailBlock  bool   `mapstructure:"auth_fail_block"`
-	SeedFile       string `mapstructure:"seed_file"`
-	MigrationsPath string `mapstructure:"migrations_path"`
-	SecretKey      string `mapstructure:"secret_key"`
+	WatchAndReload bool     `mapstructure:"reload_on_config_change"`
+	AuthFailBlock  bool     `mapstructure:"auth_fail_block"`
+	SeedFile       string   `mapstructure:"seed_file"`
+	MigrationsPath string   `mapstructure:"migrations_path"`
+	SecretKey      string   `mapstructure:"secret_key"`
+	AllowedOrigins []string `mapstructure:"cors_allowed_origins"`
+	DebugCORS      bool     `mapstructure:"cors_debug"`
 
 	Inflections map[string]string
 
@@ -205,8 +207,8 @@ func newConfig(name string) *viper.Viper {
 	vi.SetDefault("env", "development")
 
 	vi.BindEnv("env", "GO_ENV") //nolint: errcheck
-	vi.BindEnv("HOST", "HOST")  //nolint: errcheck
-	vi.BindEnv("PORT", "PORT")  //nolint: errcheck
+	vi.BindEnv("host", "HOST")  //nolint: errcheck
+	vi.BindEnv("port", "PORT")  //nolint: errcheck
 
 	vi.SetDefault("auth.rails.max_idle", 80)
 	vi.SetDefault("auth.rails.max_active", 12000)

serv/core.go

@@ -152,6 +152,10 @@ func (c *coreContext) resolvePreparedSQL() ([]byte, *stmt, error) {
 		}
 	}
 
+	if root, err = encryptCursor(ps.st.qc, root); err != nil {
+		return nil, nil, err
+	}
+
 	return root, &ps.st, nil
 }
 

serv/core_build.go

@@ -90,7 +90,7 @@ func buildMultiStmt(gql, vars []byte) ([]stmt, error) {
 	}
 
 	if len(conf.RolesQuery) == 0 {
-		return buildRoleStmt(gql, vars, "user")
+		return nil, errors.New("roles_query not defined")
 	}
 
 	stmts := make([]stmt, 0, len(conf.Roles))
@@ -99,6 +99,7 @@ func buildMultiStmt(gql, vars []byte) ([]stmt, error) {
 	for i := 0; i < len(conf.Roles); i++ {
 		role := &conf.Roles[i]
 
+		// skip anon as it's not included in the combined multi-statement
 		if role.Name == "anon" {
 			continue
 		}

serv/http.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/rs/cors"
 )
 
 const (
@@ -61,6 +63,20 @@ type resolver struct {
 	Duration    time.Duration `json:"duration"`
 }
 
+func apiV1Handler() http.Handler {
+	h := withAuth(http.HandlerFunc(apiV1), conf.Auth)
+
+	if len(conf.AllowedOrigins) != 0 {
+		c := cors.New(cors.Options{
+			AllowedOrigins:   conf.AllowedOrigins,
+			AllowCredentials: true,
+			Debug:            conf.DebugCORS,
+		})
+		h = c.Handler(h)
+	}
+
+	return h
+}
 func apiV1(w http.ResponseWriter, r *http.Request) {
 	ctx := &coreContext{Context: r.Context()}
 

serv/prepare.go

@@ -7,6 +7,7 @@ import (
 	"io"
 
 	"github.com/dosco/super-graph/allow"
+	"github.com/dosco/super-graph/psql"
 	"github.com/dosco/super-graph/qcode"
 	"github.com/jackc/pgconn"
 	"github.com/jackc/pgx/v4"
@@ -120,6 +121,9 @@ func prepareStmt(item allow.Item) error {
 			logger.Debug().Msg("Prepared statement for role: anon")
 
 			stmts2, err := buildRoleStmt(q, vars, "anon")
+			if err == psql.ErrAllTablesSkipped {
+				return nil
+			}
 			if err != nil {
 				return err
 			}

serv/reload.go

@@ -108,7 +108,11 @@ func Do(log func(string, ...interface{}), additional ...dir) error {
 				// Ensure that we use the correct events, as they are not uniform across
 				// platforms. See https://github.com/fsnotify/fsnotify/issues/74
 
-				if conf != nil && !conf.Production && strings.HasSuffix(event.Name, "/allow.list") {
+				if conf != nil && strings.HasSuffix(event.Name, "/allow.list") {
+					continue
+				}
+
+				if conf.Production {
 					continue
 				}
 

serv/serv.go

@@ -154,7 +154,7 @@ func routeHandler() (http.Handler, error) {
 
 	routes := map[string]http.Handler{
 		"/health":         http.HandlerFunc(health),
-		"/api/v1/graphql": withAuth(http.HandlerFunc(apiV1), conf.Auth),
+		"/api/v1/graphql": apiV1Handler(),
 	}
 
 	if err := setActionRoutes(routes); err != nil {

tmpl/dev.yml

@@ -36,6 +36,15 @@ migrations_path: ./config/migrations
 # encrypting the cursor data
 secret_key: supercalifajalistics
 
+# CORS: A list of origins a cross-domain request can be executed from. 
+# If the special * value is present in the list, all origins will be allowed. 
+# An origin may contain a wildcard (*) to replace 0 or more 
+# characters (i.e.: http://*.domain.com).
+cors_allowed_origins: ["*"]
+
+# Debug Cross Origin Resource Sharing requests
+cors_debug: false
+
 # Postgres related environment Variables
 # SG_DATABASE_HOST
 # SG_DATABASE_PORT
@@ -125,7 +134,9 @@ database:
 
   # Define additional variables here to be used with filters
   variables:
-    admin_account_id: "5"
+    #admin_account_id: "5"
+    admin_account_id: "sql:select id from users where admin = true limit 1"
+
 
   # Field and table names that you wish to block
   blocklist:
@@ -168,26 +179,14 @@ tables:
     table: users
 
 
-roles_query: "SELECT * FROM users WHERE id = $user_id"
+#roles_query: "SELECT * FROM users WHERE id = $user_id"
 
 roles:
   - name: anon
     tables:
-      - name: products
-        limit: 10
-
+      - name: users
         query:
-          columns: ["id", "name", "description" ]
-          aggregation: false
-
-        insert:
-          block: false
-            
-        update:
-          block: false
-
-        delete:
-          block: false
+          limit: 10
 
   - name: user
     tables:
@@ -215,8 +214,8 @@ roles:
         delete:
           block: true
 
-  - name: admin
-    match: id = 1000
-    tables:
-      - name: users
-        filters: []
+  # - name: admin
+  #   match: id = 1000
+  #   tables:
+  #     - name: users
+  #       filters: []

tmpl/prod.yml

@@ -24,7 +24,11 @@ auth_fail_block: true
 # Latency tracing for database queries and remote joins
 # the resulting latency information is returned with the
 # response
-enable_tracing: true
+enable_tracing: false
+
+# Watch the config folder and reload Super Graph
+# with the new configs when a change is detected
+reload_on_config_change: false
 
 # File that points to the database seeding script
 # seed_file: seed.js
@@ -36,6 +40,15 @@ enable_tracing: true
 # encrypting the cursor data
 # secret_key: supercalifajalistics
 
+# CORS: A list of origins a cross-domain request can be executed from. 
+# If the special * value is present in the list, all origins will be allowed. 
+# An origin may contain a wildcard (*) to replace 0 or more 
+# characters (i.e.: http://*.domain.com).
+# cors_allowed_origins: ["*"]
+
+# Debug Cross Origin Resource Sharing requests
+# cors_debug: false
+
 # Postgres related environment Variables
 # SG_DATABASE_HOST
 # SG_DATABASE_PORT
@@ -52,7 +65,7 @@ database:
   type: postgres
   host: db
   port: 5432
-  dbname: {% app_name_slug %}_development
+  dbname: {% app_name_slug %}_production
   user: postgres
   password: postgres
   #pool_size: 10