From 8b6c562ac1df829aa1afa18ddd12ed73c2dedca5 Mon Sep 17 00:00:00 2001 From: Vikram Rangnekar Date: Tue, 3 Mar 2020 13:45:47 +0530 Subject: [PATCH] Add CSV import command to seed javascript --- Dockerfile | 18 ++++++-- docs/guide.md | 6 +++ go.mod | 2 - go.sum | 4 -- rails/auth.go | 24 ++++++----- scripts/start.sh | 7 ++++ serv/cmd_seed.go | 105 +++++++++++++++++++++++++++++++++++++++++++++++ tmpl/prod.yml | 2 +- 8 files changed, 147 insertions(+), 21 deletions(-) create mode 100755 scripts/start.sh diff --git a/Dockerfile b/Dockerfile index c18834b..dc25554 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,14 +6,19 @@ RUN yarn RUN yarn build # stage: 2 -FROM golang:1.13.4-alpine as go-build +FROM golang:1.14-alpine as go-build RUN apk update && \ apk add --no-cache make && \ apk add --no-cache git && \ + apk add --no-cache jq && \ apk add --no-cache upx=3.95-r2 RUN GO111MODULE=off go get -u github.com/rafaelsq/wtc +ARG SOPS_VERSION=3.5.0 +ADD https://github.com/mozilla/sops/releases/download/v${SOPS_VERSION}/sops-v${SOPS_VERSION}.linux /usr/local/bin/sops +RUN chmod 755 /usr/local/bin/sops + WORKDIR /app COPY . /app @@ -22,9 +27,9 @@ COPY --from=react-build /web/build/ ./web/build/ RUN go mod vendor RUN make build -RUN echo "Compressing binary, will take a bit of time..." && \ - upx --ultra-brute -qq super-graph && \ - upx -t super-graph +# RUN echo "Compressing binary, will take a bit of time..." && \ +# upx --ultra-brute -qq super-graph && \ +# upx -t super-graph # stage: 3 FROM alpine:latest @@ -36,10 +41,15 @@ RUN mkdir -p /config COPY --from=go-build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=go-build /app/config/* /config/ COPY --from=go-build /app/super-graph . +COPY --from=go-build /app/scripts/start.sh . +COPY --from=go-build /usr/local/bin/sops . RUN chmod +x /super-graph +RUN chmod +x /start.sh + USER nobody EXPOSE 8080 CMD ./super-graph serv +ENTRYPOINT ["./start.sh"] diff --git a/docs/guide.md b/docs/guide.md index e62b3e2..0c939bf 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -292,6 +292,12 @@ for (i = 0; i < 10; i++) { } ``` +If you want to import a lot of data using a CSV file is the best and fastest option. The `import_csv` command uses the `COPY FROM` Postgres method to load massive amounts of data into tables. The first line of the CSV file must be the header with column names. + +```javascript +var post_count = import_csv("posts", "posts.csv") +``` + You can generate the following fake data for your seeding purposes. Below is the list of fake data functions supported by the built-in fake data library. For example `fake.image_url()` will generate a fake image url or `fake.shuffle_strings(['hello', 'world', 'cool'])` will generate a randomly shuffled version of that array of strings or `fake.rand_string(['hello', 'world', 'cool'])` will return a random string from the array provided. ``` diff --git a/go.mod b/go.mod index e1342d3..5e22811 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,6 @@ module github.com/dosco/super-graph require ( github.com/GeertJohan/go.rice v1.0.0 - github.com/Masterminds/semver v1.5.0 github.com/NYTimes/gziphandler v1.1.1 github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b @@ -12,7 +11,6 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dlclark/regexp2 v1.2.0 // indirect github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 - github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c // indirect github.com/fsnotify/fsnotify v1.4.7 github.com/garyburd/redigo v1.6.0 github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect diff --git a/go.sum b/go.sum index fd69149..06e7978 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/GeertJohan/go.incremental v1.0.0 h1:7AH+pY1XUgQE4Y1HcXYaMqAI0m9yrFqo/ github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0= github.com/GeertJohan/go.rice v1.0.0 h1:KkI6O9uMaQU3VEKaj01ulavtF7o1fWT7+pk/4voiMLQ= github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -54,8 +52,6 @@ github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 h1:cyNc40Dx5YNEO94idePU8rhVd3dn+sd04Arh0kDBAaw= github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c h1:/bXaeEuNG6V0HeyEGw11DYLW5BGsOPlcVRIXbHNUWSo= -github.com/dvyukov/go-fuzz v0.0.0-20191206100749-a378175e205c/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= diff --git a/rails/auth.go b/rails/auth.go index 98e0de7..2753302 100644 --- a/rails/auth.go +++ b/rails/auth.go @@ -4,8 +4,9 @@ import ( "encoding/json" "errors" "fmt" + "strconv" + "strings" - "github.com/Masterminds/semver" "github.com/adjust/gorails/marshal" ) @@ -37,17 +38,20 @@ func NewAuth(version, secret string) (*Auth, error) { AuthSalt: authSalt, } - ver, err := semver.NewVersion(version) - if err != nil { - return nil, fmt.Errorf("rails auth: %s", err) + var v1, v2 int + var err error + + sv := strings.Split(version, ".") + if len(sv) >= 2 { + if v1, err = strconv.Atoi(sv[0]); err != nil { + return nil, err + } + if v2, err = strconv.Atoi(sv[1]); err != nil { + return nil, err + } } - gt52, err := semver.NewConstraint(">= 5.2") - if err != nil { - return nil, fmt.Errorf("rails auth: %s", err) - } - - if gt52.Check(ver) { + if v1 >= 5 && v2 >= 2 { ra.Cipher = railsCipher52 } else { ra.Cipher = railsCipher diff --git a/scripts/start.sh b/scripts/start.sh new file mode 100755 index 0000000..b2d9439 --- /dev/null +++ b/scripts/start.sh @@ -0,0 +1,7 @@ +#!/bin/sh +if test -f "./config/$SECRETS_FILE" +then + ./sops --config ./config exec-env "./config/$SECRETS_FILE" "$*" +else + $@ +fi \ No newline at end of file diff --git a/serv/cmd_seed.go b/serv/cmd_seed.go index bb9b17b..b63a5f7 100644 --- a/serv/cmd_seed.go +++ b/serv/cmd_seed.go @@ -3,6 +3,7 @@ package serv import ( "bytes" "context" + "encoding/csv" "encoding/json" "fmt" "io" @@ -10,9 +11,12 @@ import ( "math/rand" "os" "path" + "strconv" + "strings" "github.com/brianvoe/gofakeit" "github.com/dop251/goja" + "github.com/jackc/pgx/v4" "github.com/spf13/cobra" "github.com/valyala/fasttemplate" ) @@ -42,6 +46,7 @@ func cmdDBSeed(cmd *cobra.Command, args []string) { vm := goja.New() vm.Set("graphql", graphQLFunc) + vm.Set("import_csv", importCSV) console := vm.NewObject() console.Set("log", logFunc) //nolint: errcheck @@ -129,6 +134,106 @@ func graphQLFunc(query string, data interface{}, opt map[string]string) map[stri return val } +type csvSource struct { + rows [][]string + i int +} + +func NewCSVSource(filename string) (*csvSource, error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer f.Close() + + r := csv.NewReader(f) + rows, err := r.ReadAll() + if err != nil { + return nil, err + } + + return &csvSource{rows: rows}, nil +} + +func (c *csvSource) Next() bool { + return c.i < len(c.rows) +} + +func (c *csvSource) Values() ([]interface{}, error) { + var vals []interface{} + var err error + + for _, v := range c.rows[c.i] { + switch { + case len(v) == 0: + vals = append(vals, "") + case isDigit(v): + var n int + if n, err = strconv.Atoi(v); err == nil { + vals = append(vals, n) + } + case strings.EqualFold(v, "true") || strings.EqualFold(v, "false"): + var b bool + if b, err = strconv.ParseBool(v); err == nil { + vals = append(vals, b) + } + default: + vals = append(vals, v) + } + + if err != nil { + return nil, fmt.Errorf("%w (line no %d)", err, c.i) + } + } + c.i++ + + return vals, nil +} + +func isDigit(v string) bool { + for i := range v { + if v[i] < '0' || v[i] > '9' { + return false + } + } + return true +} + +func (c *csvSource) Err() error { + return nil +} + +func importCSV(table, filename string) int64 { + if filename[0] != '/' { + filename = path.Join(confPath, filename) + } + + s, err := NewCSVSource(filename) + if err != nil { + errlog.Fatal().Err(err).Send() + } + + var cols []string + colval, _ := s.Values() + + for _, c := range colval { + cols = append(cols, c.(string)) + } + + n, err := db.CopyFrom( + context.Background(), + pgx.Identifier{table}, + cols, + s) + + if err != nil { + err = fmt.Errorf("%w (line no %d)", err, s.i) + errlog.Fatal().Err(err).Send() + } + + return n +} + //nolint: errcheck func logFunc(args ...interface{}) { for _, arg := range args { diff --git a/tmpl/prod.yml b/tmpl/prod.yml index c1e5985..d571ecc 100644 --- a/tmpl/prod.yml +++ b/tmpl/prod.yml @@ -52,7 +52,7 @@ database: type: postgres host: db port: 5432 - dbname: {% app_name_slug %}_development + dbname: {% app_name_slug %}_production user: postgres password: postgres #pool_size: 10