Add CSV import command to seed javascript

Dockerfile

@@ -6,14 +6,19 @@ RUN yarn
 RUN yarn build
 
 # stage: 2
-FROM golang:1.13.4-alpine as go-build
+FROM golang:1.14-alpine as go-build
 RUN apk update && \
     apk add --no-cache make && \
     apk add --no-cache git && \
+    apk add --no-cache jq && \
     apk add --no-cache upx=3.95-r2
 
 RUN GO111MODULE=off go get -u github.com/rafaelsq/wtc
 
+ARG SOPS_VERSION=3.5.0
+ADD https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux /usr/local/bin/sops
+RUN chmod 755 /usr/local/bin/sops
+
 WORKDIR /app
 COPY . /app
 
@@ -22,9 +27,9 @@ COPY --from=react-build /web/build/ ./web/build/
 
 RUN go mod vendor
 RUN make build
-RUN echo "Compressing binary, will take a bit of time..." && \
-  upx --ultra-brute -qq super-graph && \
-  upx -t super-graph
+# RUN echo "Compressing binary, will take a bit of time..." && \
+#   upx --ultra-brute -qq super-graph && \
+#   upx -t super-graph
 
 # stage: 3
 FROM alpine:latest
@@ -36,10 +41,15 @@ RUN mkdir -p /config
 COPY --from=go-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=go-build /app/config/* /config/
 COPY --from=go-build /app/super-graph .
+COPY --from=go-build /app/scripts/start.sh .
+COPY --from=go-build /usr/local/bin/sops .
 
 RUN chmod +x /super-graph
+RUN chmod +x /start.sh
+
 USER nobody
 
 EXPOSE 8080
 
 CMD ./super-graph serv
+ENTRYPOINT ["./start.sh"]

docs/guide.md

@@ -292,6 +292,12 @@ for (i = 0; i < 10; i++) {
 }
 ```
 
+If you want to import a lot of data using a CSV file is the best and fastest option. The `import_csv` command uses the `COPY FROM` Postgres method to load massive amounts of data into tables. The first line of the CSV file must be the header with column names.
+
+```javascript
+var post_count = import_csv("posts", "posts.csv")
+```
+
 You can generate the following fake data for your seeding purposes. Below is the list of fake data functions supported by the built-in fake data library. For example `fake.image_url()` will generate a fake image url or `fake.shuffle_strings(['hello', 'world', 'cool'])` will generate a randomly shuffled version of that array of strings or `fake.rand_string(['hello', 'world', 'cool'])` will return a random string from the array provided.
 
 ```
@@ -1975,7 +1981,7 @@ To use Yugabyte in your local development flow just uncomment the following line
 
 ## Developing Super Graph
 
-If you want to build and run Super Graph from code then the below commands will build the web ui and launch Super Graph in developer mode with a watcher to rebuild on code changes. And the demo rails app is also launched to make it essier to test changes.
+If you want to build and run Super Graph from code then the below commands will build the web ui and launch Super Graph in developer mode with a watcher to rebuild on code changes. And the demo rails app is also launched to make it easier to test changes.
 
 ```bash
 

go.mod

@@ -2,7 +2,6 @@ module github.com/dosco/super-graph
 
 require (
 	github.com/GeertJohan/go.rice v1.0.0
-	github.com/Masterminds/semver v1.5.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
@@ -12,7 +11,6 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/dlclark/regexp2 v1.2.0 // indirect
 	github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733
-	github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c // indirect
 	github.com/fsnotify/fsnotify v1.4.7
 	github.com/garyburd/redigo v1.6.0
 	github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect

go.sum

@@ -5,8 +5,6 @@ github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/
 github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
 github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ=
 github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
-github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
-github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -54,8 +52,6 @@ github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk
 github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 h1:cyNc40Dx5YNEO94idePU8rhVd3dn+sd04Arh0kDBAaw=
 github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
-github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c h1:/bXaeEuNG6V0HeyEGw11DYLW5BGsOPlcVRIXbHNUWSo=
-github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc=

psql/query.go

@@ -1127,9 +1127,15 @@ func (c *compilerContext) renderVal(ex *qcode.Exp, vars map[string]string, col *
 
 	switch ex.Type {
 	case qcode.ValVar:
-		if val, ok := vars[ex.Val]; ok {
+		val, ok := vars[ex.Val]
+		switch {
+		case ok && strings.HasPrefix(val, "sql:"):
+			io.WriteString(c.w, ` (`)
+			io.WriteString(c.w, val[4:])
+			io.WriteString(c.w, `)`)
+		case ok:
 			squoted(c.w, val)
-		} else {
+		default:
 			io.WriteString(c.w, ` '{{`)
 			io.WriteString(c.w, ex.Val)
 			io.WriteString(c.w, `}}'`)

rails/auth.go

@@ -4,8 +4,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strconv"
+	"strings"
 
-	"github.com/Masterminds/semver"
 	"github.com/adjust/gorails/marshal"
 )
 
@@ -37,17 +38,20 @@ func NewAuth(version, secret string) (*Auth, error) {
 		AuthSalt: authSalt,
 	}
 
-	ver, err := semver.NewVersion(version)
-	if err != nil {
-		return nil, fmt.Errorf("rails auth: %s", err)
+	var v1, v2 int
+	var err error
+
+	sv := strings.Split(version, ".")
+	if len(sv) >= 2 {
+		if v1, err = strconv.Atoi(sv[0]); err != nil {
+			return nil, err
+		}
+		if v2, err = strconv.Atoi(sv[1]); err != nil {
+			return nil, err
+		}
 	}
 
-	gt52, err := semver.NewConstraint(">= 5.2")
-	if err != nil {
-		return nil, fmt.Errorf("rails auth: %s", err)
-	}
-
-	if gt52.Check(ver) {
+	if v1 >= 5 && v2 >= 2 {
 		ra.Cipher = railsCipher52
 	} else {
 		ra.Cipher = railsCipher

scripts/start.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+if test -f "./config/$SECRETS_FILE"
+then
+  ./sops --config ./config exec-env "./config/$SECRETS_FILE" "$*" 
+else
+  $@
+fi

serv/args.go

@@ -57,8 +57,6 @@ func argMap(ctx context.Context, vars []byte) func(w io.Writer, tag string) (int
 			return w.Write(v1)
 		}
 
-		fmt.Println(">>>", tag, string(v))
-
 		return w.Write(escQuote(fields[0].Value))
 	}
 }

serv/cmd_seed.go

@@ -3,6 +3,7 @@ package serv
 import (
 	"bytes"
 	"context"
+	"encoding/csv"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,9 +11,12 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strconv"
+	"strings"
 
 	"github.com/brianvoe/gofakeit"
 	"github.com/dop251/goja"
+	"github.com/jackc/pgx/v4"
 	"github.com/spf13/cobra"
 	"github.com/valyala/fasttemplate"
 )
@@ -42,6 +46,7 @@ func cmdDBSeed(cmd *cobra.Command, args []string) {
 
 	vm := goja.New()
 	vm.Set("graphql", graphQLFunc)
+	vm.Set("import_csv", importCSV)
 
 	console := vm.NewObject()
 	console.Set("log", logFunc) //nolint: errcheck
@@ -129,6 +134,106 @@ func graphQLFunc(query string, data interface{}, opt map[string]string) map[stri
 	return val
 }
 
+type csvSource struct {
+	rows [][]string
+	i    int
+}
+
+func NewCSVSource(filename string) (*csvSource, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	r := csv.NewReader(f)
+	rows, err := r.ReadAll()
+	if err != nil {
+		return nil, err
+	}
+
+	return &csvSource{rows: rows}, nil
+}
+
+func (c *csvSource) Next() bool {
+	return c.i < len(c.rows)
+}
+
+func (c *csvSource) Values() ([]interface{}, error) {
+	var vals []interface{}
+	var err error
+
+	for _, v := range c.rows[c.i] {
+		switch {
+		case len(v) == 0:
+			vals = append(vals, "")
+		case isDigit(v):
+			var n int
+			if n, err = strconv.Atoi(v); err == nil {
+				vals = append(vals, n)
+			}
+		case strings.EqualFold(v, "true") || strings.EqualFold(v, "false"):
+			var b bool
+			if b, err = strconv.ParseBool(v); err == nil {
+				vals = append(vals, b)
+			}
+		default:
+			vals = append(vals, v)
+		}
+
+		if err != nil {
+			return nil, fmt.Errorf("%w (line no %d)", err, c.i)
+		}
+	}
+	c.i++
+
+	return vals, nil
+}
+
+func isDigit(v string) bool {
+	for i := range v {
+		if v[i] < '0' || v[i] > '9' {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *csvSource) Err() error {
+	return nil
+}
+
+func importCSV(table, filename string) int64 {
+	if filename[0] != '/' {
+		filename = path.Join(confPath, filename)
+	}
+
+	s, err := NewCSVSource(filename)
+	if err != nil {
+		errlog.Fatal().Err(err).Send()
+	}
+
+	var cols []string
+	colval, _ := s.Values()
+
+	for _, c := range colval {
+		cols = append(cols, c.(string))
+	}
+
+	n, err := db.CopyFrom(
+		context.Background(),
+		pgx.Identifier{table},
+		cols,
+		s)
+
+	if err != nil {
+		err = fmt.Errorf("%w (line no %d)", err, s.i)
+		errlog.Fatal().Err(err).Send()
+	}
+
+	return n
+}
+
 //nolint: errcheck
 func logFunc(args ...interface{}) {
 	for _, arg := range args {

tmpl/dev.yml

@@ -125,7 +125,9 @@ database:
 
   # Define additional variables here to be used with filters
   variables:
-    admin_account_id: "5"
+    #admin_account_id: "5"
+    admin_account_id: "sql:select id from users where admin = true limit 1"
+
 
   # Field and table names that you wish to block
   blocklist:
@@ -168,26 +170,14 @@ tables:
     table: users
 
 
-roles_query: "SELECT * FROM users WHERE id = $user_id"
+#roles_query: "SELECT * FROM users WHERE id = $user_id"
 
 roles:
   - name: anon
     tables:
-      - name: products
-        limit: 10
-
+      - name: users
         query:
-          columns: ["id", "name", "description" ]
-          aggregation: false
-
-        insert:
-          block: false
-            
-        update:
-          block: false
-
-        delete:
-          block: false
+          limit: 10
 
   - name: user
     tables:
@@ -215,8 +205,8 @@ roles:
         delete:
           block: true
 
-  - name: admin
-    match: id = 1000
-    tables:
-      - name: users
-        filters: []
+  # - name: admin
+  #   match: id = 1000
+  #   tables:
+  #     - name: users
+  #       filters: []

tmpl/prod.yml

@@ -52,7 +52,7 @@ database:
   type: postgres
   host: db
   port: 5432
-  dbname: {% app_name_slug %}_development
+  dbname: {% app_name_slug %}_production
   user: postgres
   password: postgres
   #pool_size: 10