diff --git a/config/seed.js b/config/seed.js new file mode 100644 index 0000000..a1c5cd9 --- /dev/null +++ b/config/seed.js @@ -0,0 +1,114 @@ +var user_count = 10 + customer_count = 100 + product_count = 50 + purchase_count = 100 + +var users = [] + customers = [] + products = [] + +for (i = 0; i < user_count; i++) { + var pwd = fake.password() + var data = { + full_name: fake.name(), + avatar: fake.image_url(), + phone: fake.phone(), + email: fake.email(), + password: pwd, + password_confirmation: pwd, + created_at: "now", + updated_at: "now" + } + + var res = graphql(" \ + mutation { \ + user(insert: $data) { \ + id \ + } \ + }", { data: data }) + + users.push(res.user) +} + +for (i = 0; i < product_count; i++) { + var n = Math.floor(Math.random() * users.length) + var user = users[n] + + var desc = [ + fake.beer_style(), + fake.beer_hop(), + fake.beer_yeast(), + fake.beer_ibu(), + fake.beer_alcohol(), + fake.beer_blg(), + ].join(", ") + + var data = { + name: fake.beer_name(), + description: desc, + price: fake.price(), + user_id: user.id, + created_at: "now", + updated_at: "now" + } + + var res = graphql(" \ + mutation { \ + product(insert: $data) { \ + id \ + } \ + }", { data: data }) + products.push(res.product) +} + +for (i = 0; i < customer_count; i++) { + var pwd = fake.password() + + var data = { + stripe_id: "CUS-" + fake.uuid(), + full_name: fake.name(), + phone: fake.phone(), + email: fake.email(), + password: pwd, + password_confirmation: pwd, + created_at: "now", + updated_at: "now" + } + + var res = graphql(" \ + mutation { \ + customer(insert: $data) { \ + id \ + } \ + }", { data: data }) + customers.push(res.customer) +} + +for (i = 0; i < purchase_count; i++) { + var sale_type = fake.rand_string(["rented", "bought"]) + + if (sale_type === "rented") { + var due_date = fake.date() + var returned = fake.date() + } + + var data = { + customer_id: customers[Math.floor(Math.random() * customer_count)].id, + product_id: products[Math.floor(Math.random() * product_count)].id, + sale_type: sale_type, + quantity: Math.floor(Math.random() * 10), + due_date: due_date, + returned: returned, + created_at: "now", + updated_at: "now" + } + + var res = graphql(" \ + mutation { \ + purchase(insert: $data) { \ + id \ + } \ + }", { data: data }) + + console.log(res) +} \ No newline at end of file diff --git a/fuzzbuzz.yaml b/fuzzbuzz.yaml index 3a18da6..fbf4622 100644 --- a/fuzzbuzz.yaml +++ b/fuzzbuzz.yaml @@ -1,8 +1,9 @@ base: ubuntu:16.04 targets: + - name: qcode language: go - version: "1.12" + version: "1.13" corpus: ./corpus memory_limit: "100" # in megabytes timeout: "500" # in milliseconds @@ -13,3 +14,17 @@ targets: # the repository will be cloned to # $GOPATH/src/github.com/fuzzbuzz/tutorial checkout: github.com/dosco/super-graph + + - name: jsn + language: go + version: "1.13" + corpus: ./corpus + memory_limit: "100" # in megabytes + timeout: "500" # in milliseconds + harness: + function: FuzzerEntrypoint + # package defines where to import FuzzerEntrypoint from + package: github.com/dosco/super-graph/jsn + # the repository will be cloned to + # $GOPATH/src/github.com/fuzzbuzz/tutorial + checkout: github.com/dosco/super-graph \ No newline at end of file diff --git a/go.mod b/go.mod index 1105b95..7c169bd 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,16 @@ require ( github.com/OneOfOne/xxhash v1.2.5 // indirect github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 + github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/cespare/xxhash/v2 v2.0.0 github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/dlclark/regexp2 v1.2.0 // indirect + github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 github.com/fsnotify/fsnotify v1.4.7 github.com/garyburd/redigo v1.6.0 github.com/go-pg/pg v8.0.1+incompatible + github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/gobuffalo/flect v0.1.1 github.com/gorilla/websocket v1.4.0 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect diff --git a/go.sum b/go.sum index c76d338..ced36a8 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3/go.mod h1:FlkD11Rtg github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= +github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= +github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/cespare/xxhash/v2 v2.0.0 h1:Eb1IiuHmi3FhT12NKfqCQXSXRqc4NTMvgJoREemrSt4= github.com/cespare/xxhash/v2 v2.0.0/go.mod h1:MaMeaVDXZNmTpkOyhVs3/WfjgobkbQgfrVnrr3DyZL0= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -21,12 +23,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 h1:cyNc40Dx5YNEO94idePU8rhVd3dn+sd04Arh0kDBAaw= +github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/go-pg/pg v8.0.1+incompatible h1:gi93AxXmqlFGT0os5z2kTnbDqCk6BHXnA9MMApVxAkY= github.com/go-pg/pg v8.0.1+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/gobuffalo/flect v0.1.1 h1:GTZJjJufv9FxgRs1+0Soo3wj+Md3kTUmTER/YE4uINA= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= diff --git a/jsn/corpus/001.json b/jsn/corpus/001.json new file mode 100644 index 0000000..2e2449e --- /dev/null +++ b/jsn/corpus/001.json @@ -0,0 +1,97 @@ +{ + "data": { + "test": { "__twitter_id": "ABCD" }, + "users": [ + { + "id": 1, + "full_name": "Sidney Stroman", + "email": "user0@demo.com", + "__twitter_id": "2048666903444506956", + "embed": { + "id": 8, + "full_name": "Caroll Orn Sr.", + "email": "joannarau@hegmann.io", + "__twitter_id": "ABC123" + "more": [{ + "__twitter_id": "more123", + "hello: "world + }] + } + }, + { + "id": 2, + "full_name": "Jerry Dickinson", + "email": "user1@demo.com", + "__twitter_id": [{ "name": "hello" }, { "name": "world"}] + }, + { + "id": 3, + "full_name": "Kenna Cassin", + "email": "user2@demo.com", + "__twitter_id": { "name": "hello", "address": { "work": "1 infinity loop" } } + }, + { + "id": 4, + "full_name": "Mr. Pat Parisian", + "email": "__twitter_id", + "__twitter_id": 1234567890 + }, + { + "id": 5, + "full_name": "Bette Ebert", + "email": "janeenrath@goyette.com", + "__twitter_id": 1.23E + }, + { + "id": 6, + "full_name": "Everett Kiehn", + "email": "michael@bartoletti.com", + "__twitter_id": true + }, + { + "id": 7, + "full_name": "Katrina Cronin", + "email": "loretaklocko@framivolkman.org", + "__twitter_id": false + }, + { + "id": 8, + "full_name": "Caroll Orn Sr.", + "email": "joannarau@hegmann.io", + "__twitter_id": "2048666903444506956" + }, + { + "id": 9, + "full_name": "Gwendolyn Ziemann", + "email": "renaytoy@rutherford.co", + "__twitter_id": ["hello", "world"] + }, + { + "id": 10, + "full_name": "Mrs. Rosann Fritsch", + "email": "holliemosciski@thiel.org", + "__twitter_id": "2048666903444506956" + }, + { + "id": 11, + "full_name": "Arden Koss", + "email": "cristobalankunding@howewelch.org", + "__twitter_id": "2048666903444506956", + "something": null + }, + { + "id": 12, + "full_name": "Brenton Bauch PhD", + "email": "renee@miller.co", + "__twitter_id": 1 + }, + { + "id": 13, + "full_name": "Daine Gleichner", + "email": "andrea@gmail.com", + "__twitter_id": "", + "id__twitter_id": "NOOO", + "work_email": "andrea@nienow.co" + } + ]} + } \ No newline at end of file diff --git a/jsn/fuzz.go b/jsn/fuzz.go new file mode 100644 index 0000000..6a0b21f --- /dev/null +++ b/jsn/fuzz.go @@ -0,0 +1,35 @@ +package jsn + +import "bytes" + +// FuzzerEntrypoint for Fuzzbuzz +func FuzzerEntryPoint(data []byte) int { + err1 := Validate(string(data)) + + var b1 bytes.Buffer + err2 := Filter(&b1, data, []string{"id", "full_name", "embed"}) + + path1 := [][]byte{[]byte("data"), []byte("users")} + Strip(data, path1) + + from := []Field{ + {[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)}, + {[]byte("__twitter_id"), []byte(`"ABC123"`)}, + } + + to := []Field{ + {[]byte("__twitter_id"), []byte(`"1234567890"`)}, + {[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)}, + } + + var b2 bytes.Buffer + err3 := Replace(&b2, data, from, to) + + Keys(data) + + if err1 != nil || err2 != nil || err3 != nil { + return -1 + } + + return 0 +} diff --git a/psql/insert.go b/psql/insert.go index 131ee4c..d040607 100644 --- a/psql/insert.go +++ b/psql/insert.go @@ -60,10 +60,12 @@ func (c *compilerContext) renderInsert(qc *qcode.QCode, w *bytes.Buffer, vars Va } c.w.WriteString(`WITH `) - c.w.WriteString(root.Table) + quoted(c.w, ti.Name) - c.w.WriteString(` AS (WITH input AS (SELECT {{insert}}::json AS j) INSERT INTO `) - c.w.WriteString(root.Table) + c.w.WriteString(` AS (WITH "input" AS (SELECT {{`) + c.w.WriteString(root.ActionVar) + c.w.WriteString(`}}::json AS j) INSERT INTO `) + c.w.WriteString(ti.Name) io.WriteString(c.w, ` (`) c.renderInsertUpdateColumns(qc, w, jt, ti) io.WriteString(c.w, `)`) @@ -79,7 +81,7 @@ func (c *compilerContext) renderInsert(qc *qcode.QCode, w *bytes.Buffer, vars Va } c.w.WriteString(`(NULL::`) - c.w.WriteString(root.Table) + c.w.WriteString(ti.Name) c.w.WriteString(`, i.j) t RETURNING *) `) return 0, nil @@ -122,10 +124,12 @@ func (c *compilerContext) renderUpdate(qc *qcode.QCode, w *bytes.Buffer, vars Va } c.w.WriteString(`WITH `) - c.w.WriteString(root.Table) + quoted(c.w, ti.Name) - c.w.WriteString(` AS (WITH input AS (SELECT {{update}}::json AS j) UPDATE `) - c.w.WriteString(root.Table) + c.w.WriteString(` AS (WITH "input" AS (SELECT {{`) + c.w.WriteString(root.ActionVar) + c.w.WriteString(`}}::json AS j) UPDATE `) + c.w.WriteString(ti.Name) io.WriteString(c.w, ` SET (`) c.renderInsertUpdateColumns(qc, w, jt, ti) @@ -140,7 +144,7 @@ func (c *compilerContext) renderUpdate(qc *qcode.QCode, w *bytes.Buffer, vars Va } c.w.WriteString(`(NULL::`) - c.w.WriteString(root.Table) + c.w.WriteString(ti.Name) c.w.WriteString(`, i.j) t)`) io.WriteString(c.w, ` WHERE `) @@ -163,7 +167,7 @@ func (c *compilerContext) renderDelete(qc *qcode.QCode, w *bytes.Buffer, vars Va } c.w.WriteString(`DELETE FROM `) - c.w.WriteString(root.Table) + c.w.WriteString(ti.Name) io.WriteString(c.w, ` WHERE `) if err := c.renderWhere(root, ti); err != nil { @@ -174,3 +178,9 @@ func (c *compilerContext) renderDelete(qc *qcode.QCode, w *bytes.Buffer, vars Va return 0, nil } + +func quoted(w *bytes.Buffer, identifier string) { + w.WriteString(`"`) + w.WriteString(identifier) + w.WriteString(`"`) +} diff --git a/psql/select.go b/psql/select.go index 0518c9b..b0f0e91 100644 --- a/psql/select.go +++ b/psql/select.go @@ -79,6 +79,11 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w *bytes.Buffer) (uint32, erro c := &compilerContext{w, qc.Selects, co} root := &qc.Selects[0] + ti, err := c.schema.GetTable(root.Table) + if err != nil { + return 0, err + } + st := NewStack() st.Push(root.ID + closeBlock) st.Push(root.ID) @@ -88,7 +93,13 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w *bytes.Buffer) (uint32, erro c.w.WriteString(`SELECT json_object_agg('`) c.w.WriteString(root.FieldName) c.w.WriteString(`', `) - c.w.WriteString(root.Table) + + if ti.Singular == false { + c.w.WriteString(root.Table) + } else { + c.w.WriteString("sel_json_") + int2string(c.w, root.ID) + } c.w.WriteString(`) FROM (`) var ignored uint32 @@ -209,7 +220,8 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint if ti.Singular == false { //fmt.Fprintf(w, `SELECT coalesce(json_agg("%s"`, c.sel.Table) c.w.WriteString(`SELECT coalesce(json_agg("`) - c.w.WriteString(sel.Table) + c.w.WriteString("sel_json_") + int2string(c.w, sel.ID) c.w.WriteString(`"`) if hasOrder { @@ -255,7 +267,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint //fmt.Fprintf(w, `)) AS "%s"`, c.sel.Table) c.w.WriteString(`))`) - alias(c.w, sel.Table) + aliasWithID(c.w, "sel_json", sel.ID) // END-ROW-TO-JSON if hasOrder { @@ -304,9 +316,9 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select, ti *DBTableInfo) } if ti.Singular == false { - //fmt.Fprintf(w, `) AS "%s_%d"`, c.sel.Table, c.sel.ID) + //fmt.Fprintf(w, `) AS "sel_json_agg_%d"`, c.sel.ID) c.w.WriteString(`)`) - aliasWithID(c.w, sel.Table, sel.ID) + aliasWithID(c.w, "sel_json_agg", sel.ID) } return nil diff --git a/serv/allow.go b/serv/allow.go index 0fec2ca..a0f347c 100644 --- a/serv/allow.go +++ b/serv/allow.go @@ -27,12 +27,14 @@ type allowList struct { list map[string]*allowItem filepath string saveChan chan *allowItem + active bool } func initAllowList(path string) { _allowList = allowList{ list: make(map[string]*allowItem), saveChan: make(chan *allowItem), + active: true, } if len(path) != 0 { @@ -79,7 +81,7 @@ func initAllowList(path string) { } func (al *allowList) add(req *gqlReq) { - if len(req.ref) == 0 || len(req.Query) == 0 { + if al.active == false || len(req.ref) == 0 || len(req.Query) == 0 { return } @@ -91,6 +93,10 @@ func (al *allowList) add(req *gqlReq) { } func (al *allowList) load() { + if al.active == false { + return + } + b, err := ioutil.ReadFile(al.filepath) if err != nil { log.Fatal(err) @@ -168,6 +174,9 @@ func (al *allowList) load() { } func (al *allowList) save(item *allowItem) { + if al.active == false { + return + } al.list[gqlHash(item.gql, item.vars)] = item f, err := os.Create(al.filepath) diff --git a/serv/cmd.go b/serv/cmd.go new file mode 100644 index 0000000..663fd1f --- /dev/null +++ b/serv/cmd.go @@ -0,0 +1,204 @@ +package serv + +import ( + "errors" + "flag" + "fmt" + "os" + "strings" + + "github.com/dosco/super-graph/psql" + "github.com/dosco/super-graph/qcode" + "github.com/go-pg/pg" + "github.com/gobuffalo/flect" + "github.com/rs/zerolog" + "github.com/spf13/viper" +) + +//go:generate esc -o static.go -ignore \\.DS_Store -prefix ../web/build -private -pkg serv ../web/build + +const ( + serverName = "Super Graph" + + authFailBlockAlways = iota + 1 + authFailBlockPerQuery + authFailBlockNever +) + +var ( + logger *zerolog.Logger + conf *config + db *pg.DB + qcompile *qcode.Compiler + pcompile *psql.Compiler + authFailBlock int +) + +func initLog() *zerolog.Logger { + logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). + With(). + Timestamp(). + Caller(). + Logger() + + return &logger + /* + log := logrus.New() + logger.Formatter = new(logrus.TextFormatter) + logger.Formatter.(*logrus.TextFormatter).DisableColors = false + logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true + logger.Level = logrus.TraceLevel + logger.Out = os.Stdout + */ +} + +func initConf(path string) (*config, error) { + vi := viper.New() + + vi.SetEnvPrefix("SG") + vi.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + vi.AutomaticEnv() + + vi.AddConfigPath(path) + vi.AddConfigPath("./config") + vi.SetConfigName(getConfigName()) + + vi.SetDefault("host_port", "0.0.0.0:8080") + vi.SetDefault("web_ui", false) + vi.SetDefault("enable_tracing", false) + vi.SetDefault("auth_fail_block", "always") + vi.SetDefault("seed_file", "seed.js") + + vi.SetDefault("database.type", "postgres") + vi.SetDefault("database.host", "localhost") + vi.SetDefault("database.port", 5432) + vi.SetDefault("database.user", "postgres") + + vi.SetDefault("env", "development") + vi.BindEnv("env", "GO_ENV") + vi.BindEnv("HOST", "HOST") + vi.BindEnv("PORT", "PORT") + + vi.SetDefault("auth.rails.max_idle", 80) + vi.SetDefault("auth.rails.max_active", 12000) + + if err := vi.ReadInConfig(); err != nil { + return nil, err + } + + c := &config{} + + if err := vi.Unmarshal(c); err != nil { + return nil, fmt.Errorf("unable to decode config, %v", err) + } + + for k, v := range c.Inflections { + flect.AddPlural(k, v) + } + + for i := range c.DB.Tables { + t := c.DB.Tables[i] + t.Name = flect.Pluralize(strings.ToLower(t.Name)) + } + + authFailBlock = getAuthFailBlock(c) + + //fmt.Printf("%#v", c) + + return c, nil +} + +func initDB(c *config) (*pg.DB, error) { + opt := &pg.Options{ + Addr: strings.Join([]string{c.DB.Host, c.DB.Port}, ":"), + User: c.DB.User, + Password: c.DB.Password, + Database: c.DB.DBName, + ApplicationName: c.AppName, + } + + if c.DB.PoolSize != 0 { + opt.PoolSize = conf.DB.PoolSize + } + + if c.DB.MaxRetries != 0 { + opt.MaxRetries = c.DB.MaxRetries + } + + if len(c.DB.Schema) != 0 { + opt.OnConnect = func(conn *pg.Conn) error { + _, err := conn.Exec("set search_path=?", c.DB.Schema) + if err != nil { + return err + } + return nil + } + } + + db := pg.Connect(opt) + if db == nil { + return nil, errors.New("failed to connect to postgres db") + } + + return db, nil +} + +func Init() { + var err error + + path := flag.String("path", "./config", "Path to config files") + flag.Parse() + + logger = initLog() + + conf, err = initConf(*path) + if err != nil { + logger.Fatal().Err(err).Msg("failed to read config") + } + + logLevel, err := zerolog.ParseLevel(conf.LogLevel) + if err != nil { + logger.Error().Err(err).Msg("error setting log_level") + } + zerolog.SetGlobalLevel(logLevel) + + db, err = initDB(conf) + if err != nil { + logger.Fatal().Err(err).Msg("failed to connect to database") + } + + qcompile, pcompile, err = initCompilers(conf) + if err != nil { + logger.Fatal().Err(err).Msg("failed to connect to database") + } + + if err := initResolvers(); err != nil { + logger.Fatal().Err(err).Msg("failed to initialized resolvers") + } + + args := flag.Args() + + if len(args) == 0 { + cmdServ(*path) + } + + switch args[0] { + case "seed": + cmdSeed(*path) + + case "serv": + fallthrough + + default: + logger.Fatal().Msg("options: [serve|seed]") + } + +} + +func cmdServ(path string) { + initAllowList(path) + initPreparedList() + initWatcher(path) + + startHTTP() +} diff --git a/serv/config.go b/serv/config.go index 92887c6..e4a4a0f 100644 --- a/serv/config.go +++ b/serv/config.go @@ -18,6 +18,7 @@ type config struct { UseAllowList bool `mapstructure:"use_allow_list"` WatchAndReload bool `mapstructure:"reload_on_config_change"` AuthFailBlock string `mapstructure:"auth_fail_block"` + SeedFile string `mapstructure:"seed_file"` Inflections map[string]string Auth struct { diff --git a/serv/core.go b/serv/core.go index 1d26ccb..aaf27da 100644 --- a/serv/core.go +++ b/serv/core.go @@ -31,21 +31,29 @@ type coreContext struct { } func (c *coreContext) handleReq(w io.Writer, req *http.Request) error { + c.req.ref = req.Referer() + c.req.hdr = req.Header + + b, err := c.execQuery() + if err != nil { + return err + } + + return c.render(w, b) +} + +func (c *coreContext) execQuery() ([]byte, error) { var err error var skipped uint32 var qc *qcode.QCode var data []byte - c.req.ref = req.Referer() - - //conf.UseAllowList = true - if conf.UseAllowList { var ps *preparedItem data, ps, err = c.resolvePreparedSQL(c.req.Query) if err != nil { - return err + return nil, err } skipped = ps.skipped @@ -55,17 +63,17 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error { qc, err = qcompile.Compile([]byte(c.req.Query)) if err != nil { - return err + return nil, err } data, skipped, err = c.resolveSQL(qc) if err != nil { - return err + return nil, err } } if len(data) == 0 || skipped == 0 { - return c.render(w, data) + return data, nil } sel := qc.Selects @@ -83,31 +91,31 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error { var to []jsn.Field switch { case len(from) == 1: - to, err = c.resolveRemote(req, h, from[0], sel, sfmap) + to, err = c.resolveRemote(c.req.hdr, h, from[0], sel, sfmap) case len(from) > 1: - to, err = c.resolveRemotes(req, h, from, sel, sfmap) + to, err = c.resolveRemotes(c.req.hdr, h, from, sel, sfmap) default: - return errors.New("something wrong no remote ids found in db response") + return nil, errors.New("something wrong no remote ids found in db response") } if err != nil { - return err + return nil, err } var ob bytes.Buffer err = jsn.Replace(&ob, data, from, to) if err != nil { - return err + return nil, err } - return c.render(w, ob.Bytes()) + return ob.Bytes(), nil } func (c *coreContext) resolveRemote( - req *http.Request, + hdr http.Header, h *xxhash.Digest, field jsn.Field, sel []qcode.Select, @@ -143,7 +151,7 @@ func (c *coreContext) resolveRemote( st := time.Now() - b, err := r.Fn(req, id) + b, err := r.Fn(hdr, id) if err != nil { return nil, err } @@ -173,7 +181,7 @@ func (c *coreContext) resolveRemote( } func (c *coreContext) resolveRemotes( - req *http.Request, + hdr http.Header, h *xxhash.Digest, from []jsn.Field, sel []qcode.Select, @@ -218,7 +226,7 @@ func (c *coreContext) resolveRemotes( st := time.Now() - b, err := r.Fn(req, id) + b, err := r.Fn(hdr, id) if err != nil { cerr = fmt.Errorf("%s: %s", s.Table, err) return @@ -324,6 +332,7 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode) ( if conf.LogLevel == "debug" { os.Stdout.WriteString(finalSQL) + os.Stdout.WriteString("\n\n") } var st time.Time @@ -346,7 +355,7 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode) ( } } - fmt.Printf("RAW: %#v\n", finalSQL) + //fmt.Printf("\nRAW: %#v\n", finalSQL) var root json.RawMessage _, err = tx.QueryOne(pg.Scan(&root), finalSQL) diff --git a/serv/http.go b/serv/http.go index 270e563..c943110 100644 --- a/serv/http.go +++ b/serv/http.go @@ -30,6 +30,7 @@ type gqlReq struct { Query string `json:"query"` Vars json.RawMessage `json:"variables"` ref string + hdr http.Header } type variables map[string]json.RawMessage diff --git a/serv/reso.go b/serv/reso.go index 2706447..402d381 100644 --- a/serv/reso.go +++ b/serv/reso.go @@ -19,7 +19,7 @@ var ( type resolvFn struct { IDField []byte Path [][]byte - Fn func(r *http.Request, id []byte) ([]byte, error) + Fn func(h http.Header, id []byte) ([]byte, error) } func initResolvers() error { @@ -92,11 +92,11 @@ func initRemotes(t configTable) error { return nil } -func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { +func buildFn(r configRemote) func(http.Header, []byte) ([]byte, error) { reqURL := strings.Replace(r.URL, "$id", "%s", 1) client := &http.Client{} - fn := func(inReq *http.Request, id []byte) ([]byte, error) { + fn := func(hdr http.Header, id []byte) ([]byte, error) { uri := fmt.Sprintf(reqURL, id) req, err := http.NewRequest("GET", uri, nil) if err != nil { @@ -108,10 +108,10 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { } for _, v := range r.PassHeaders { - req.Header.Set(v, inReq.Header.Get(v)) + req.Header.Set(v, hdr.Get(v)) } - if host, ok := req.Header["Host"]; ok { + if host, ok := hdr["Host"]; ok { req.Host = host[0] } diff --git a/serv/seed.go b/serv/seed.go new file mode 100644 index 0000000..19732d7 --- /dev/null +++ b/serv/seed.go @@ -0,0 +1,260 @@ +package serv + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/brianvoe/gofakeit" + "github.com/dop251/goja" +) + +func cmdSeed(cpath string) { + conf.UseAllowList = false + + b, err := ioutil.ReadFile(path.Join(cpath, conf.SeedFile)) + if err != nil { + logger.Fatal().Err(err).Msg("failed to read file") + } + + vm := goja.New() + vm.Set("graphql", graphQLFunc) + + console := vm.NewObject() + console.Set("log", logFunc) + vm.Set("console", console) + + fake := vm.NewObject() + setFakeFuncs(fake) + vm.Set("fake", fake) + + _, err = vm.RunScript("seed.js", string(b)) + if err != nil { + logger.Fatal().Err(err).Msg("failed to execute script") + } +} + +//func runFunc(call goja.FunctionCall) { +func graphQLFunc(query string, data interface{}) map[string]interface{} { + b, err := json.Marshal(data) + if err != nil { + logger.Fatal().Err(err).Msg("failed to json serialize") + } + + c := &coreContext{Context: context.Background()} + c.req.Query = query + c.req.Vars = b + + res, err := c.execQuery() + if err != nil { + logger.Fatal().Err(err).Msg("graphql query failed") + } + + val := make(map[string]interface{}) + + err = json.Unmarshal(res, &val) + if err != nil { + logger.Fatal().Err(err).Msg("failed to deserialize json") + } + + return val +} + +func logFunc(args ...interface{}) { + for _, arg := range args { + if _, ok := arg.(map[string]interface{}); ok { + j, err := json.MarshalIndent(arg, "", " ") + if err != nil { + continue + } + os.Stdout.Write(j) + } else { + io.WriteString(os.Stdout, fmt.Sprintf("%v", arg)) + } + + io.WriteString(os.Stdout, "\n") + } +} + +func setFakeFuncs(f *goja.Object) { + gofakeit.Seed(0) + + // Person + f.Set("person", gofakeit.Person) + f.Set("name", gofakeit.Name) + f.Set("name_prefix", gofakeit.NamePrefix) + f.Set("name_suffix", gofakeit.NameSuffix) + f.Set("first_name", gofakeit.FirstName) + f.Set("last_name", gofakeit.LastName) + f.Set("gender", gofakeit.Gender) + f.Set("ssn", gofakeit.SSN) + f.Set("contact", gofakeit.Contact) + f.Set("email", gofakeit.Email) + f.Set("phone", gofakeit.Phone) + f.Set("phone_formatted", gofakeit.PhoneFormatted) + f.Set("username", gofakeit.Username) + f.Set("password", gofakeit.Password) + + // Address + f.Set("address", gofakeit.Address) + f.Set("city", gofakeit.City) + f.Set("country", gofakeit.Country) + f.Set("country_abr", gofakeit.CountryAbr) + f.Set("state", gofakeit.State) + f.Set("state_abr", gofakeit.StateAbr) + f.Set("status_code", gofakeit.StatusCode) + f.Set("street", gofakeit.Street) + f.Set("street_name", gofakeit.StreetName) + f.Set("street_number", gofakeit.StreetNumber) + f.Set("street_prefix", gofakeit.StreetPrefix) + f.Set("street_suffix", gofakeit.StreetSuffix) + f.Set("zip", gofakeit.Zip) + f.Set("latitude", gofakeit.Latitude) + f.Set("latitude_in_range", gofakeit.LatitudeInRange) + f.Set("longitude", gofakeit.Longitude) + f.Set("longitude_in_range", gofakeit.LongitudeInRange) + + // Beer + f.Set("beer_alcohol", gofakeit.BeerAlcohol) + f.Set("beer_hop", gofakeit.BeerHop) + f.Set("beer_ibu", gofakeit.BeerIbu) + f.Set("beer_blg", gofakeit.BeerBlg) + f.Set("beer_malt", gofakeit.BeerMalt) + f.Set("beer_name", gofakeit.BeerName) + f.Set("beer_style", gofakeit.BeerStyle) + f.Set("beer_yeast", gofakeit.BeerYeast) + + // Cars + f.Set("vehicle", gofakeit.Vehicle) + f.Set("vehicle_type", gofakeit.VehicleType) + f.Set("car_maker", gofakeit.CarMaker) + f.Set("car_model", gofakeit.CarModel) + f.Set("fuel_type", gofakeit.FuelType) + f.Set("transmission_gear_type", gofakeit.TransmissionGearType) + + // Text + + f.Set("word", gofakeit.Word) + f.Set("sentence", gofakeit.Sentence) + f.Set("paragrph", gofakeit.Paragraph) + f.Set("question", gofakeit.Question) + f.Set("quote", gofakeit.Quote) + + // Misc + f.Set("generate", gofakeit.Generate) + f.Set("boolean", gofakeit.Bool) + f.Set("uuid", gofakeit.UUID) + + // Colors + f.Set("color", gofakeit.Color) + f.Set("hex_color", gofakeit.HexColor) + f.Set("rgb_color", gofakeit.RGBColor) + f.Set("safe_color", gofakeit.SafeColor) + + // Internet + f.Set("url", gofakeit.URL) + f.Set("image_url", gofakeit.ImageURL) + f.Set("domain_name", gofakeit.DomainName) + f.Set("domain_suffix", gofakeit.DomainSuffix) + f.Set("ipv4_address", gofakeit.IPv4Address) + f.Set("ipv6_address", gofakeit.IPv6Address) + f.Set("simple_status_code", gofakeit.SimpleStatusCode) + f.Set("http_method", gofakeit.HTTPMethod) + f.Set("user_agent", gofakeit.UserAgent) + f.Set("user_agent_firefox", gofakeit.FirefoxUserAgent) + f.Set("user_agent_chrome", gofakeit.ChromeUserAgent) + f.Set("user_agent_opera", gofakeit.OperaUserAgent) + f.Set("user_agent_safari", gofakeit.SafariUserAgent) + + // Date / Time + f.Set("date", gofakeit.Date) + f.Set("date_range", gofakeit.DateRange) + f.Set("nano_second", gofakeit.NanoSecond) + f.Set("second", gofakeit.Second) + f.Set("minute", gofakeit.Minute) + f.Set("hour", gofakeit.Hour) + f.Set("month", gofakeit.Month) + f.Set("day", gofakeit.Day) + f.Set("weekday", gofakeit.WeekDay) + f.Set("year", gofakeit.Year) + f.Set("timezone", gofakeit.TimeZone) + f.Set("timezone_abv", gofakeit.TimeZoneAbv) + f.Set("timezone_full", gofakeit.TimeZoneFull) + f.Set("timezone_offset", gofakeit.TimeZoneOffset) + + // Payment + f.Set("price", gofakeit.Price) + f.Set("credit_card", gofakeit.CreditCard) + f.Set("credit_card_cvv", gofakeit.CreditCardCvv) + f.Set("credit_card_number", gofakeit.CreditCardNumber) + f.Set("credit_card_number_luhn", gofakeit.CreditCardNumberLuhn) + f.Set("credit_card_type", gofakeit.CreditCardType) + f.Set("currency", gofakeit.Currency) + f.Set("currency_long", gofakeit.CurrencyLong) + f.Set("currency_short", gofakeit.CurrencyShort) + + // Company + f.Set("bs", gofakeit.BS) + f.Set("buzzword", gofakeit.BuzzWord) + f.Set("company", gofakeit.Company) + f.Set("company_suffix", gofakeit.CompanySuffix) + f.Set("job", gofakeit.Job) + f.Set("job_description", gofakeit.JobDescriptor) + f.Set("job_level", gofakeit.JobLevel) + f.Set("job_title", gofakeit.JobTitle) + + // Hacker + f.Set("hacker_abbreviation", gofakeit.HackerAbbreviation) + f.Set("hacker_adjective", gofakeit.HackerAdjective) + f.Set("hacker_ingverb", gofakeit.HackerIngverb) + f.Set("hacker_noun", gofakeit.HackerNoun) + f.Set("hacker_phrase", gofakeit.HackerPhrase) + f.Set("hacker_verb", gofakeit.HackerVerb) + + //Hipster + f.Set("hipster_word", gofakeit.HipsterWord) + f.Set("hipster_paragraph", gofakeit.HipsterParagraph) + f.Set("hipster_sentence", gofakeit.HipsterSentence) + + //Languages + //f.Set("language", gofakeit.Language) + //f.Set("language_abbreviation", gofakeit.LanguageAbbreviation) + //f.Set("language_abbreviation", gofakeit.LanguageAbbreviation) + + // File + f.Set("extension", gofakeit.Extension) + f.Set("mine_type", gofakeit.MimeType) + + // Numbers + f.Set("number", gofakeit.Number) + f.Set("numerify", gofakeit.Numerify) + f.Set("int8", gofakeit.Int8) + f.Set("int16", gofakeit.Int16) + f.Set("int32", gofakeit.Int32) + f.Set("int64", gofakeit.Int64) + f.Set("uint8", gofakeit.Uint8) + f.Set("uint16", gofakeit.Uint16) + f.Set("uint32", gofakeit.Uint32) + f.Set("uint64", gofakeit.Uint64) + f.Set("float32", gofakeit.Float32) + f.Set("float32_range", gofakeit.Float32Range) + f.Set("float64", gofakeit.Float64) + f.Set("float64_range", gofakeit.Float64Range) + f.Set("shuffle_ints", gofakeit.ShuffleInts) + f.Set("mac_address", gofakeit.MacAddress) + + // String + f.Set("digit", gofakeit.Digit) + f.Set("letter", gofakeit.Letter) + f.Set("lexify", gofakeit.Lexify) + f.Set("rand_string", gofakeit.RandString) + f.Set("shuffle_strings", gofakeit.ShuffleStrings) + f.Set("numerify", gofakeit.Numerify) + + //f.Set("programming_language", gofakeit.ProgrammingLanguage) + +} diff --git a/serv/serv.go b/serv/serv.go index 40bcf4b..54f79a9 100644 --- a/serv/serv.go +++ b/serv/serv.go @@ -2,8 +2,6 @@ package serv import ( "context" - "errors" - "flag" "fmt" "net/http" "os" @@ -13,139 +11,8 @@ import ( "github.com/dosco/super-graph/psql" "github.com/dosco/super-graph/qcode" - "github.com/go-pg/pg" - "github.com/gobuffalo/flect" - "github.com/rs/zerolog" - "github.com/spf13/viper" ) -//go:generate esc -o static.go -ignore \\.DS_Store -prefix ../web/build -private -pkg serv ../web/build - -const ( - serverName = "Super Graph" - - authFailBlockAlways = iota + 1 - authFailBlockPerQuery - authFailBlockNever -) - -var ( - logger *zerolog.Logger - conf *config - db *pg.DB - qcompile *qcode.Compiler - pcompile *psql.Compiler - authFailBlock int -) - -func initLog() *zerolog.Logger { - logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}). - With(). - Timestamp(). - Caller(). - Logger() - - return &logger - /* - log := logrus.New() - logger.Formatter = new(logrus.TextFormatter) - logger.Formatter.(*logrus.TextFormatter).DisableColors = false - logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true - logger.Level = logrus.TraceLevel - logger.Out = os.Stdout - */ -} - -func initConf(path string) (*config, error) { - vi := viper.New() - - vi.SetEnvPrefix("SG") - vi.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - vi.AutomaticEnv() - - vi.AddConfigPath(path) - vi.AddConfigPath("./config") - vi.SetConfigName(getConfigName()) - - vi.SetDefault("host_port", "0.0.0.0:8080") - vi.SetDefault("web_ui", false) - vi.SetDefault("enable_tracing", false) - vi.SetDefault("auth_fail_block", "always") - - vi.SetDefault("database.type", "postgres") - vi.SetDefault("database.host", "localhost") - vi.SetDefault("database.port", 5432) - vi.SetDefault("database.user", "postgres") - - vi.SetDefault("env", "development") - vi.BindEnv("env", "GO_ENV") - vi.BindEnv("HOST", "HOST") - vi.BindEnv("PORT", "PORT") - - vi.SetDefault("auth.rails.max_idle", 80) - vi.SetDefault("auth.rails.max_active", 12000) - - if err := vi.ReadInConfig(); err != nil { - return nil, err - } - - c := &config{} - - if err := vi.Unmarshal(c); err != nil { - return nil, fmt.Errorf("unable to decode config, %v", err) - } - - for k, v := range c.Inflections { - flect.AddPlural(k, v) - } - - for i := range c.DB.Tables { - t := c.DB.Tables[i] - t.Name = flect.Pluralize(strings.ToLower(t.Name)) - } - - authFailBlock = getAuthFailBlock(c) - - //fmt.Printf("%#v", c) - - return c, nil -} - -func initDB(c *config) (*pg.DB, error) { - opt := &pg.Options{ - Addr: strings.Join([]string{c.DB.Host, c.DB.Port}, ":"), - User: c.DB.User, - Password: c.DB.Password, - Database: c.DB.DBName, - ApplicationName: c.AppName, - } - - if c.DB.PoolSize != 0 { - opt.PoolSize = conf.DB.PoolSize - } - - if c.DB.MaxRetries != 0 { - opt.MaxRetries = c.DB.MaxRetries - } - - if len(c.DB.Schema) != 0 { - opt.OnConnect = func(conn *pg.Conn) error { - _, err := conn.Exec("set search_path=?", c.DB.Schema) - if err != nil { - return err - } - return nil - } - } - - db := pg.Connect(opt) - if db == nil { - return nil, errors.New("failed to connect to postgres db") - } - - return db, nil -} - func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) { schema, err := psql.NewDBSchema(db, c.getAliasMap()) if err != nil { @@ -191,46 +58,6 @@ func initWatcher(path string) { }() } -func Init() { - var err error - - path := flag.String("path", "./", "Path to config files") - flag.Parse() - - logger = initLog() - - conf, err = initConf(*path) - if err != nil { - logger.Fatal().Err(err).Msg("failed to read config") - } - - logLevel, err := zerolog.ParseLevel(conf.LogLevel) - if err != nil { - logger.Error().Err(err).Msg("error setting log_level") - } - zerolog.SetGlobalLevel(logLevel) - - db, err = initDB(conf) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to database") - } - - qcompile, pcompile, err = initCompilers(conf) - if err != nil { - logger.Fatal().Err(err).Msg("failed to connect to database") - } - - if err := initResolvers(); err != nil { - logger.Fatal().Err(err).Msg("failed to initialized resolvers") - } - - initAllowList(*path) - initPreparedList() - initWatcher(*path) - - startHTTP() -} - func startHTTP() { hp := strings.SplitN(conf.HostPort, ":", 2)