wpetit/super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add database seeding capability

Browse Source
This commit is contained in:
Vikram Rangnekar 2019-09-20 00:19:11 -04:00
parent 1a949859e9
commit 18ce030056
16 changed files with 819 additions and 213 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

114
config/seed.js Normal file
View File

@ -0,0 +1,114 @@
var user_count = 10
customer_count = 100
product_count = 50
purchase_count = 100
var users = []
customers = []
products = []
for (i = 0; i < user_count; i++) {
var pwd = fake.password()
var data = {
full_name: fake.name(),
avatar: fake.image_url(),
phone: fake.phone(),
email: fake.email(),
password: pwd,
password_confirmation: pwd,
created_at: "now",
updated_at: "now"
}
var res = graphql(" \
mutation { \
user(insert: $data) { \
id \
} \
}", { data: data })
users.push(res.user)
}
for (i = 0; i < product_count; i++) {
var n = Math.floor(Math.random() * users.length)
var user = users[n]
var desc = [
fake.beer_style(),
fake.beer_hop(),
fake.beer_yeast(),
fake.beer_ibu(),
fake.beer_alcohol(),
fake.beer_blg(),
].join(", ")
var data = {
name: fake.beer_name(),
description: desc,
price: fake.price(),
user_id: user.id,
created_at: "now",
updated_at: "now"
}
var res = graphql(" \
mutation { \
product(insert: $data) { \
id \
} \
}", { data: data })
products.push(res.product)
}
for (i = 0; i < customer_count; i++) {
var pwd = fake.password()
var data = {
stripe_id: "CUS-" + fake.uuid(),
full_name: fake.name(),
phone: fake.phone(),
email: fake.email(),
password: pwd,
password_confirmation: pwd,
created_at: "now",
updated_at: "now"
}
var res = graphql(" \
mutation { \
customer(insert: $data) { \
id \
} \
}", { data: data })
customers.push(res.customer)
}
for (i = 0; i < purchase_count; i++) {
var sale_type = fake.rand_string(["rented", "bought"])
if (sale_type === "rented") {
var due_date = fake.date()
var returned = fake.date()
}
var data = {
customer_id: customers[Math.floor(Math.random() * customer_count)].id,
product_id: products[Math.floor(Math.random() * product_count)].id,
sale_type: sale_type,
quantity: Math.floor(Math.random() * 10),
due_date: due_date,
returned: returned,
created_at: "now",
updated_at: "now"
}
var res = graphql(" \
mutation { \
purchase(insert: $data) { \
id \
} \
}", { data: data })
console.log(res)
}

17
fuzzbuzz.yaml
View File

@ -1,8 +1,9 @@
base: ubuntu:16.04
targets:
- name: qcode
language: go
version: "1.12"
version: "1.13"
corpus: ./corpus
memory_limit: "100" # in megabytes
timeout: "500" # in milliseconds
@ -13,3 +14,17 @@ targets:
# the repository will be cloned to
# $GOPATH/src/github.com/fuzzbuzz/tutorial
checkout: github.com/dosco/super-graph
- name: jsn
language: go
version: "1.13"
corpus: ./corpus
memory_limit: "100" # in megabytes
timeout: "500" # in milliseconds
harness:
function: FuzzerEntrypoint
# package defines where to import FuzzerEntrypoint from
package: github.com/dosco/super-graph/jsn
# the repository will be cloned to
# $GOPATH/src/github.com/fuzzbuzz/tutorial
checkout: github.com/dosco/super-graph

4
go.mod
View File

@ -6,12 +6,16 @@ require (
github.com/OneOfOne/xxhash v1.2.5 // indirect
github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3
github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737
github.com/brianvoe/gofakeit v3.18.0+incompatible
github.com/cespare/xxhash/v2 v2.0.0
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dlclark/regexp2 v1.2.0 // indirect
github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733
github.com/fsnotify/fsnotify v1.4.7
github.com/garyburd/redigo v1.6.0
github.com/go-pg/pg v8.0.1+incompatible
github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect
github.com/gobuffalo/flect v0.1.1
github.com/gorilla/websocket v1.4.0
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect

8
go.sum
View File

@ -10,6 +10,8 @@ github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3/go.mod h1:FlkD11Rtg
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g=
github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=
github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8=
github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc=
github.com/cespare/xxhash/v2 v2.0.0 h1:Eb1IiuHmi3FhT12NKfqCQXSXRqc4NTMvgJoREemrSt4=
github.com/cespare/xxhash/v2 v2.0.0/go.mod h1:MaMeaVDXZNmTpkOyhVs3/WfjgobkbQgfrVnrr3DyZL0=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@ -21,12 +23,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk=
github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733 h1:cyNc40Dx5YNEO94idePU8rhVd3dn+sd04Arh0kDBAaw=
github.com/dop251/goja v0.0.0-20190912223329-aa89e6a4c733/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc=
github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/go-pg/pg v8.0.1+incompatible h1:gi93AxXmqlFGT0os5z2kTnbDqCk6BHXnA9MMApVxAkY=
github.com/go-pg/pg v8.0.1+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug=
github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/gobuffalo/flect v0.1.1 h1:GTZJjJufv9FxgRs1+0Soo3wj+Md3kTUmTER/YE4uINA=
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=

97
jsn/corpus/001.json Normal file
View File

@ -0,0 +1,97 @@
{
"data": {
"test": { "__twitter_id": "ABCD" },
"users": [
{
"id": 1,
"full_name": "Sidney Stroman",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"embed": {
"id": 8,
"full_name": "Caroll Orn Sr.",
"email": "joannarau@hegmann.io",
"__twitter_id": "ABC123"
"more": [{
"__twitter_id": "more123",
"hello: "world
}]
}
},
{
"id": 2,
"full_name": "Jerry Dickinson",
"email": "user1@demo.com",
"__twitter_id": [{ "name": "hello" }, { "name": "world"}]
},
{
"id": 3,
"full_name": "Kenna Cassin",
"email": "user2@demo.com",
"__twitter_id": { "name": "hello", "address": { "work": "1 infinity loop" } }
},
{
"id": 4,
"full_name": "Mr. Pat Parisian",
"email": "__twitter_id",
"__twitter_id": 1234567890
},
{
"id": 5,
"full_name": "Bette Ebert",
"email": "janeenrath@goyette.com",
"__twitter_id": 1.23E
},
{
"id": 6,
"full_name": "Everett Kiehn",
"email": "michael@bartoletti.com",
"__twitter_id": true
},
{
"id": 7,
"full_name": "Katrina Cronin",
"email": "loretaklocko@framivolkman.org",
"__twitter_id": false
},
{
"id": 8,
"full_name": "Caroll Orn Sr.",
"email": "joannarau@hegmann.io",
"__twitter_id": "2048666903444506956"
},
{
"id": 9,
"full_name": "Gwendolyn Ziemann",
"email": "renaytoy@rutherford.co",
"__twitter_id": ["hello", "world"]
},
{
"id": 10,
"full_name": "Mrs. Rosann Fritsch",
"email": "holliemosciski@thiel.org",
"__twitter_id": "2048666903444506956"
},
{
"id": 11,
"full_name": "Arden Koss",
"email": "cristobalankunding@howewelch.org",
"__twitter_id": "2048666903444506956",
"something": null
},
{
"id": 12,
"full_name": "Brenton Bauch PhD",
"email": "renee@miller.co",
"__twitter_id": 1
},
{
"id": 13,
"full_name": "Daine Gleichner",
"email": "andrea@gmail.com",
"__twitter_id": "",
"id__twitter_id": "NOOO",
"work_email": "andrea@nienow.co"
}
]}
}

35
jsn/fuzz.go Normal file
View File

@ -0,0 +1,35 @@
package jsn
import "bytes"
// FuzzerEntrypoint for Fuzzbuzz
func FuzzerEntryPoint(data []byte) int {
err1 := Validate(string(data))
var b1 bytes.Buffer
err2 := Filter(&b1, data, []string{"id", "full_name", "embed"})
path1 := [][]byte{[]byte("data"), []byte("users")}
Strip(data, path1)
from := []Field{
{[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)},
{[]byte("__twitter_id"), []byte(`"ABC123"`)},
}
to := []Field{
{[]byte("__twitter_id"), []byte(`"1234567890"`)},
{[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)},
}
var b2 bytes.Buffer
err3 := Replace(&b2, data, from, to)
Keys(data)
if err1 != nil || err2 != nil || err3 != nil {
return -1
}
return 0
}

28
psql/insert.go
View File

@ -60,10 +60,12 @@ func (c *compilerContext) renderInsert(qc *qcode.QCode, w *bytes.Buffer, vars Va
}
c.w.WriteString(`WITH `)
c.w.WriteString(root.Table)
quoted(c.w, ti.Name)
c.w.WriteString(` AS (WITH input AS (SELECT {{insert}}::json AS j) INSERT INTO `)
c.w.WriteString(root.Table)
c.w.WriteString(` AS (WITH "input" AS (SELECT {{`)
c.w.WriteString(root.ActionVar)
c.w.WriteString(`}}::json AS j) INSERT INTO `)
c.w.WriteString(ti.Name)
io.WriteString(c.w, ` (`)
c.renderInsertUpdateColumns(qc, w, jt, ti)
io.WriteString(c.w, `)`)
@ -79,7 +81,7 @@ func (c *compilerContext) renderInsert(qc *qcode.QCode, w *bytes.Buffer, vars Va
}
c.w.WriteString(`(NULL::`)
c.w.WriteString(root.Table)
c.w.WriteString(ti.Name)
c.w.WriteString(`, i.j) t RETURNING *) `)
return 0, nil
@ -122,10 +124,12 @@ func (c *compilerContext) renderUpdate(qc *qcode.QCode, w *bytes.Buffer, vars Va
}
c.w.WriteString(`WITH `)
c.w.WriteString(root.Table)
quoted(c.w, ti.Name)
c.w.WriteString(` AS (WITH input AS (SELECT {{update}}::json AS j) UPDATE `)
c.w.WriteString(root.Table)
c.w.WriteString(` AS (WITH "input" AS (SELECT {{`)
c.w.WriteString(root.ActionVar)
c.w.WriteString(`}}::json AS j) UPDATE `)
c.w.WriteString(ti.Name)
io.WriteString(c.w, ` SET (`)
c.renderInsertUpdateColumns(qc, w, jt, ti)
@ -140,7 +144,7 @@ func (c *compilerContext) renderUpdate(qc *qcode.QCode, w *bytes.Buffer, vars Va
}
c.w.WriteString(`(NULL::`)
c.w.WriteString(root.Table)
c.w.WriteString(ti.Name)
c.w.WriteString(`, i.j) t)`)
io.WriteString(c.w, ` WHERE `)
@ -163,7 +167,7 @@ func (c *compilerContext) renderDelete(qc *qcode.QCode, w *bytes.Buffer, vars Va
}
c.w.WriteString(`DELETE FROM `)
c.w.WriteString(root.Table)
c.w.WriteString(ti.Name)
io.WriteString(c.w, ` WHERE `)
if err := c.renderWhere(root, ti); err != nil {
@ -174,3 +178,9 @@ func (c *compilerContext) renderDelete(qc *qcode.QCode, w *bytes.Buffer, vars Va
return 0, nil
}
func quoted(w *bytes.Buffer, identifier string) {
w.WriteString(`"`)
w.WriteString(identifier)
w.WriteString(`"`)
}

22
psql/select.go
View File

@ -79,6 +79,11 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w *bytes.Buffer) (uint32, erro
c := &compilerContext{w, qc.Selects, co}
root := &qc.Selects[0]
ti, err := c.schema.GetTable(root.Table)
if err != nil {
return 0, err
}
st := NewStack()
st.Push(root.ID + closeBlock)
st.Push(root.ID)
@ -88,7 +93,13 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w *bytes.Buffer) (uint32, erro
c.w.WriteString(`SELECT json_object_agg('`)
c.w.WriteString(root.FieldName)
c.w.WriteString(`', `)
c.w.WriteString(root.Table)
if ti.Singular == false {
c.w.WriteString(root.Table)
} else {
c.w.WriteString("sel_json_")
int2string(c.w, root.ID)
}
c.w.WriteString(`) FROM (`)
var ignored uint32
@ -209,7 +220,8 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint
if ti.Singular == false {
//fmt.Fprintf(w, `SELECT coalesce(json_agg("%s"`, c.sel.Table)
c.w.WriteString(`SELECT coalesce(json_agg("`)
c.w.WriteString(sel.Table)
c.w.WriteString("sel_json_")
int2string(c.w, sel.ID)
c.w.WriteString(`"`)
if hasOrder {
@ -255,7 +267,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint
//fmt.Fprintf(w, `)) AS "%s"`, c.sel.Table)
c.w.WriteString(`))`)
alias(c.w, sel.Table)
aliasWithID(c.w, "sel_json", sel.ID)
// END-ROW-TO-JSON
if hasOrder {
@ -304,9 +316,9 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select, ti *DBTableInfo)
}
if ti.Singular == false {
//fmt.Fprintf(w, `) AS "%s_%d"`, c.sel.Table, c.sel.ID)
//fmt.Fprintf(w, `) AS "sel_json_agg_%d"`, c.sel.ID)
c.w.WriteString(`)`)
aliasWithID(c.w, sel.Table, sel.ID)
aliasWithID(c.w, "sel_json_agg", sel.ID)
}
return nil

11
serv/allow.go
View File

@ -27,12 +27,14 @@ type allowList struct {
list map[string]*allowItem
filepath string
saveChan chan *allowItem
active bool
}
func initAllowList(path string) {
_allowList = allowList{
list: make(map[string]*allowItem),
saveChan: make(chan *allowItem),
active: true,
}
if len(path) != 0 {
@ -79,7 +81,7 @@ func initAllowList(path string) {
}
func (al *allowList) add(req *gqlReq) {
if len(req.ref) == 0 || len(req.Query) == 0 {
if al.active == false || len(req.ref) == 0 || len(req.Query) == 0 {
return
}
@ -91,6 +93,10 @@ func (al *allowList) add(req *gqlReq) {
}
func (al *allowList) load() {
if al.active == false {
return
}
b, err := ioutil.ReadFile(al.filepath)
if err != nil {
log.Fatal(err)
@ -168,6 +174,9 @@ func (al *allowList) load() {
}
func (al *allowList) save(item *allowItem) {
if al.active == false {
return
}
al.list[gqlHash(item.gql, item.vars)] = item
f, err := os.Create(al.filepath)

204
serv/cmd.go Normal file
View File

@ -0,0 +1,204 @@
package serv
import (
"errors"
"flag"
"fmt"
"os"
"strings"
"github.com/dosco/super-graph/psql"
"github.com/dosco/super-graph/qcode"
"github.com/go-pg/pg"
"github.com/gobuffalo/flect"
"github.com/rs/zerolog"
"github.com/spf13/viper"
)
//go:generate esc -o static.go -ignore \\.DS_Store -prefix ../web/build -private -pkg serv ../web/build
const (
serverName = "Super Graph"
authFailBlockAlways = iota + 1
authFailBlockPerQuery
authFailBlockNever
)
var (
logger *zerolog.Logger
conf *config
db *pg.DB
qcompile *qcode.Compiler
pcompile *psql.Compiler
authFailBlock int
)
func initLog() *zerolog.Logger {
logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
With().
Timestamp().
Caller().
Logger()
return &logger
/*
log := logrus.New()
logger.Formatter = new(logrus.TextFormatter)
logger.Formatter.(*logrus.TextFormatter).DisableColors = false
logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true
logger.Level = logrus.TraceLevel
logger.Out = os.Stdout
*/
}
func initConf(path string) (*config, error) {
vi := viper.New()
vi.SetEnvPrefix("SG")
vi.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
vi.AutomaticEnv()
vi.AddConfigPath(path)
vi.AddConfigPath("./config")
vi.SetConfigName(getConfigName())
vi.SetDefault("host_port", "0.0.0.0:8080")
vi.SetDefault("web_ui", false)
vi.SetDefault("enable_tracing", false)
vi.SetDefault("auth_fail_block", "always")
vi.SetDefault("seed_file", "seed.js")
vi.SetDefault("database.type", "postgres")
vi.SetDefault("database.host", "localhost")
vi.SetDefault("database.port", 5432)
vi.SetDefault("database.user", "postgres")
vi.SetDefault("env", "development")
vi.BindEnv("env", "GO_ENV")
vi.BindEnv("HOST", "HOST")
vi.BindEnv("PORT", "PORT")
vi.SetDefault("auth.rails.max_idle", 80)
vi.SetDefault("auth.rails.max_active", 12000)
if err := vi.ReadInConfig(); err != nil {
return nil, err
}
c := &config{}
if err := vi.Unmarshal(c); err != nil {
return nil, fmt.Errorf("unable to decode config, %v", err)
}
for k, v := range c.Inflections {
flect.AddPlural(k, v)
}
for i := range c.DB.Tables {
t := c.DB.Tables[i]
t.Name = flect.Pluralize(strings.ToLower(t.Name))
}
authFailBlock = getAuthFailBlock(c)
//fmt.Printf("%#v", c)
return c, nil
}
func initDB(c *config) (*pg.DB, error) {
opt := &pg.Options{
Addr: strings.Join([]string{c.DB.Host, c.DB.Port}, ":"),
User: c.DB.User,
Password: c.DB.Password,
Database: c.DB.DBName,
ApplicationName: c.AppName,
}
if c.DB.PoolSize != 0 {
opt.PoolSize = conf.DB.PoolSize
}
if c.DB.MaxRetries != 0 {
opt.MaxRetries = c.DB.MaxRetries
}
if len(c.DB.Schema) != 0 {
opt.OnConnect = func(conn *pg.Conn) error {
_, err := conn.Exec("set search_path=?", c.DB.Schema)
if err != nil {
return err
}
return nil
}
}
db := pg.Connect(opt)
if db == nil {
return nil, errors.New("failed to connect to postgres db")
}
return db, nil
}
func Init() {
var err error
path := flag.String("path", "./config", "Path to config files")
flag.Parse()
logger = initLog()
conf, err = initConf(*path)
if err != nil {
logger.Fatal().Err(err).Msg("failed to read config")
}
logLevel, err := zerolog.ParseLevel(conf.LogLevel)
if err != nil {
logger.Error().Err(err).Msg("error setting log_level")
}
zerolog.SetGlobalLevel(logLevel)
db, err = initDB(conf)
if err != nil {
logger.Fatal().Err(err).Msg("failed to connect to database")
}
qcompile, pcompile, err = initCompilers(conf)
if err != nil {
logger.Fatal().Err(err).Msg("failed to connect to database")
}
if err := initResolvers(); err != nil {
logger.Fatal().Err(err).Msg("failed to initialized resolvers")
}
args := flag.Args()
if len(args) == 0 {
cmdServ(*path)
}
switch args[0] {
case "seed":
cmdSeed(*path)
case "serv":
fallthrough
default:
logger.Fatal().Msg("options: [serve|seed]")
}
}
func cmdServ(path string) {
initAllowList(path)
initPreparedList()
initWatcher(path)
startHTTP()
}

1
serv/config.go
View File

@ -18,6 +18,7 @@ type config struct {
UseAllowList bool `mapstructure:"use_allow_list"`
WatchAndReload bool `mapstructure:"reload_on_config_change"`
AuthFailBlock string `mapstructure:"auth_fail_block"`
SeedFile string `mapstructure:"seed_file"`
Inflections map[string]string
Auth struct {

47
serv/core.go
View File

@ -31,21 +31,29 @@ type coreContext struct {
}
func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
c.req.ref = req.Referer()
c.req.hdr = req.Header
b, err := c.execQuery()
if err != nil {
return err
}
return c.render(w, b)
}
func (c *coreContext) execQuery() ([]byte, error) {
var err error
var skipped uint32
var qc *qcode.QCode
var data []byte
c.req.ref = req.Referer()
//conf.UseAllowList = true
if conf.UseAllowList {
var ps *preparedItem
data, ps, err = c.resolvePreparedSQL(c.req.Query)
if err != nil {
return err
return nil, err
}
skipped = ps.skipped
@ -55,17 +63,17 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
qc, err = qcompile.Compile([]byte(c.req.Query))
if err != nil {
return err
return nil, err
}
data, skipped, err = c.resolveSQL(qc)
if err != nil {
return err
return nil, err
}
}
if len(data) == 0 || skipped == 0 {
return c.render(w, data)
return data, nil
}
sel := qc.Selects
@ -83,31 +91,31 @@ func (c *coreContext) handleReq(w io.Writer, req *http.Request) error {
var to []jsn.Field
switch {
case len(from) == 1:
to, err = c.resolveRemote(req, h, from[0], sel, sfmap)
to, err = c.resolveRemote(c.req.hdr, h, from[0], sel, sfmap)
case len(from) > 1:
to, err = c.resolveRemotes(req, h, from, sel, sfmap)
to, err = c.resolveRemotes(c.req.hdr, h, from, sel, sfmap)
default:
return errors.New("something wrong no remote ids found in db response")
return nil, errors.New("something wrong no remote ids found in db response")
}
if err != nil {
return err
return nil, err
}
var ob bytes.Buffer
err = jsn.Replace(&ob, data, from, to)
if err != nil {
return err
return nil, err
}
return c.render(w, ob.Bytes())
return ob.Bytes(), nil
}
func (c *coreContext) resolveRemote(
req *http.Request,
hdr http.Header,
h *xxhash.Digest,
field jsn.Field,
sel []qcode.Select,
@ -143,7 +151,7 @@ func (c *coreContext) resolveRemote(
st := time.Now()
b, err := r.Fn(req, id)
b, err := r.Fn(hdr, id)
if err != nil {
return nil, err
}
@ -173,7 +181,7 @@ func (c *coreContext) resolveRemote(
}
func (c *coreContext) resolveRemotes(
req *http.Request,
hdr http.Header,
h *xxhash.Digest,
from []jsn.Field,
sel []qcode.Select,
@ -218,7 +226,7 @@ func (c *coreContext) resolveRemotes(
st := time.Now()
b, err := r.Fn(req, id)
b, err := r.Fn(hdr, id)
if err != nil {
cerr = fmt.Errorf("%s: %s", s.Table, err)
return
@ -324,6 +332,7 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode) (
if conf.LogLevel == "debug" {
os.Stdout.WriteString(finalSQL)
os.Stdout.WriteString("\n\n")
}
var st time.Time
@ -346,7 +355,7 @@ func (c *coreContext) resolveSQL(qc *qcode.QCode) (
}
}
fmt.Printf("RAW: %#v\n", finalSQL)
//fmt.Printf("\nRAW: %#v\n", finalSQL)
var root json.RawMessage
_, err = tx.QueryOne(pg.Scan(&root), finalSQL)

1
serv/http.go
View File

@ -30,6 +30,7 @@ type gqlReq struct {
Query string `json:"query"`
Vars json.RawMessage `json:"variables"`
ref string
hdr http.Header
}
type variables map[string]json.RawMessage

10
serv/reso.go
View File

@ -19,7 +19,7 @@ var (
type resolvFn struct {
IDField []byte
Path [][]byte
Fn func(r *http.Request, id []byte) ([]byte, error)
Fn func(h http.Header, id []byte) ([]byte, error)
}
func initResolvers() error {
@ -92,11 +92,11 @@ func initRemotes(t configTable) error {
return nil
}
func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
func buildFn(r configRemote) func(http.Header, []byte) ([]byte, error) {
reqURL := strings.Replace(r.URL, "$id", "%s", 1)
client := &http.Client{}
fn := func(inReq *http.Request, id []byte) ([]byte, error) {
fn := func(hdr http.Header, id []byte) ([]byte, error) {
uri := fmt.Sprintf(reqURL, id)
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
@ -108,10 +108,10 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
}
for _, v := range r.PassHeaders {
req.Header.Set(v, inReq.Header.Get(v))
req.Header.Set(v, hdr.Get(v))
}
if host, ok := req.Header["Host"]; ok {
if host, ok := hdr["Host"]; ok {
req.Host = host[0]
}

260
serv/seed.go Normal file
View File

@ -0,0 +1,260 @@
package serv
import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"github.com/brianvoe/gofakeit"
"github.com/dop251/goja"
)
func cmdSeed(cpath string) {
conf.UseAllowList = false
b, err := ioutil.ReadFile(path.Join(cpath, conf.SeedFile))
if err != nil {
logger.Fatal().Err(err).Msg("failed to read file")
}
vm := goja.New()
vm.Set("graphql", graphQLFunc)
console := vm.NewObject()
console.Set("log", logFunc)
vm.Set("console", console)
fake := vm.NewObject()
setFakeFuncs(fake)
vm.Set("fake", fake)
_, err = vm.RunScript("seed.js", string(b))
if err != nil {
logger.Fatal().Err(err).Msg("failed to execute script")
}
}
//func runFunc(call goja.FunctionCall) {
func graphQLFunc(query string, data interface{}) map[string]interface{} {
b, err := json.Marshal(data)
if err != nil {
logger.Fatal().Err(err).Msg("failed to json serialize")
}
c := &coreContext{Context: context.Background()}
c.req.Query = query
c.req.Vars = b
res, err := c.execQuery()
if err != nil {
logger.Fatal().Err(err).Msg("graphql query failed")
}
val := make(map[string]interface{})
err = json.Unmarshal(res, &val)
if err != nil {
logger.Fatal().Err(err).Msg("failed to deserialize json")
}
return val
}
func logFunc(args ...interface{}) {
for _, arg := range args {
if _, ok := arg.(map[string]interface{}); ok {
j, err := json.MarshalIndent(arg, "", " ")
if err != nil {
continue
}
os.Stdout.Write(j)
} else {
io.WriteString(os.Stdout, fmt.Sprintf("%v", arg))
}
io.WriteString(os.Stdout, "\n")
}
}
func setFakeFuncs(f *goja.Object) {
gofakeit.Seed(0)
// Person
f.Set("person", gofakeit.Person)
f.Set("name", gofakeit.Name)
f.Set("name_prefix", gofakeit.NamePrefix)
f.Set("name_suffix", gofakeit.NameSuffix)
f.Set("first_name", gofakeit.FirstName)
f.Set("last_name", gofakeit.LastName)
f.Set("gender", gofakeit.Gender)
f.Set("ssn", gofakeit.SSN)
f.Set("contact", gofakeit.Contact)
f.Set("email", gofakeit.Email)
f.Set("phone", gofakeit.Phone)
f.Set("phone_formatted", gofakeit.PhoneFormatted)
f.Set("username", gofakeit.Username)
f.Set("password", gofakeit.Password)
// Address
f.Set("address", gofakeit.Address)
f.Set("city", gofakeit.City)
f.Set("country", gofakeit.Country)
f.Set("country_abr", gofakeit.CountryAbr)
f.Set("state", gofakeit.State)
f.Set("state_abr", gofakeit.StateAbr)
f.Set("status_code", gofakeit.StatusCode)
f.Set("street", gofakeit.Street)
f.Set("street_name", gofakeit.StreetName)
f.Set("street_number", gofakeit.StreetNumber)
f.Set("street_prefix", gofakeit.StreetPrefix)
f.Set("street_suffix", gofakeit.StreetSuffix)
f.Set("zip", gofakeit.Zip)
f.Set("latitude", gofakeit.Latitude)
f.Set("latitude_in_range", gofakeit.LatitudeInRange)
f.Set("longitude", gofakeit.Longitude)
f.Set("longitude_in_range", gofakeit.LongitudeInRange)
// Beer
f.Set("beer_alcohol", gofakeit.BeerAlcohol)
f.Set("beer_hop", gofakeit.BeerHop)
f.Set("beer_ibu", gofakeit.BeerIbu)
f.Set("beer_blg", gofakeit.BeerBlg)
f.Set("beer_malt", gofakeit.BeerMalt)
f.Set("beer_name", gofakeit.BeerName)
f.Set("beer_style", gofakeit.BeerStyle)
f.Set("beer_yeast", gofakeit.BeerYeast)
// Cars
f.Set("vehicle", gofakeit.Vehicle)
f.Set("vehicle_type", gofakeit.VehicleType)
f.Set("car_maker", gofakeit.CarMaker)
f.Set("car_model", gofakeit.CarModel)
f.Set("fuel_type", gofakeit.FuelType)
f.Set("transmission_gear_type", gofakeit.TransmissionGearType)
// Text
f.Set("word", gofakeit.Word)
f.Set("sentence", gofakeit.Sentence)
f.Set("paragrph", gofakeit.Paragraph)
f.Set("question", gofakeit.Question)
f.Set("quote", gofakeit.Quote)
// Misc
f.Set("generate", gofakeit.Generate)
f.Set("boolean", gofakeit.Bool)
f.Set("uuid", gofakeit.UUID)
// Colors
f.Set("color", gofakeit.Color)
f.Set("hex_color", gofakeit.HexColor)
f.Set("rgb_color", gofakeit.RGBColor)
f.Set("safe_color", gofakeit.SafeColor)
// Internet
f.Set("url", gofakeit.URL)
f.Set("image_url", gofakeit.ImageURL)
f.Set("domain_name", gofakeit.DomainName)
f.Set("domain_suffix", gofakeit.DomainSuffix)
f.Set("ipv4_address", gofakeit.IPv4Address)
f.Set("ipv6_address", gofakeit.IPv6Address)
f.Set("simple_status_code", gofakeit.SimpleStatusCode)
f.Set("http_method", gofakeit.HTTPMethod)
f.Set("user_agent", gofakeit.UserAgent)
f.Set("user_agent_firefox", gofakeit.FirefoxUserAgent)
f.Set("user_agent_chrome", gofakeit.ChromeUserAgent)
f.Set("user_agent_opera", gofakeit.OperaUserAgent)
f.Set("user_agent_safari", gofakeit.SafariUserAgent)
// Date / Time
f.Set("date", gofakeit.Date)
f.Set("date_range", gofakeit.DateRange)
f.Set("nano_second", gofakeit.NanoSecond)
f.Set("second", gofakeit.Second)
f.Set("minute", gofakeit.Minute)
f.Set("hour", gofakeit.Hour)
f.Set("month", gofakeit.Month)
f.Set("day", gofakeit.Day)
f.Set("weekday", gofakeit.WeekDay)
f.Set("year", gofakeit.Year)
f.Set("timezone", gofakeit.TimeZone)
f.Set("timezone_abv", gofakeit.TimeZoneAbv)
f.Set("timezone_full", gofakeit.TimeZoneFull)
f.Set("timezone_offset", gofakeit.TimeZoneOffset)
// Payment
f.Set("price", gofakeit.Price)
f.Set("credit_card", gofakeit.CreditCard)
f.Set("credit_card_cvv", gofakeit.CreditCardCvv)
f.Set("credit_card_number", gofakeit.CreditCardNumber)
f.Set("credit_card_number_luhn", gofakeit.CreditCardNumberLuhn)
f.Set("credit_card_type", gofakeit.CreditCardType)
f.Set("currency", gofakeit.Currency)
f.Set("currency_long", gofakeit.CurrencyLong)
f.Set("currency_short", gofakeit.CurrencyShort)
// Company
f.Set("bs", gofakeit.BS)
f.Set("buzzword", gofakeit.BuzzWord)
f.Set("company", gofakeit.Company)
f.Set("company_suffix", gofakeit.CompanySuffix)
f.Set("job", gofakeit.Job)
f.Set("job_description", gofakeit.JobDescriptor)
f.Set("job_level", gofakeit.JobLevel)
f.Set("job_title", gofakeit.JobTitle)
// Hacker
f.Set("hacker_abbreviation", gofakeit.HackerAbbreviation)
f.Set("hacker_adjective", gofakeit.HackerAdjective)
f.Set("hacker_ingverb", gofakeit.HackerIngverb)
f.Set("hacker_noun", gofakeit.HackerNoun)
f.Set("hacker_phrase", gofakeit.HackerPhrase)
f.Set("hacker_verb", gofakeit.HackerVerb)
//Hipster
f.Set("hipster_word", gofakeit.HipsterWord)
f.Set("hipster_paragraph", gofakeit.HipsterParagraph)
f.Set("hipster_sentence", gofakeit.HipsterSentence)
//Languages
//f.Set("language", gofakeit.Language)
//f.Set("language_abbreviation", gofakeit.LanguageAbbreviation)
//f.Set("language_abbreviation", gofakeit.LanguageAbbreviation)
// File
f.Set("extension", gofakeit.Extension)
f.Set("mine_type", gofakeit.MimeType)
// Numbers
f.Set("number", gofakeit.Number)
f.Set("numerify", gofakeit.Numerify)
f.Set("int8", gofakeit.Int8)
f.Set("int16", gofakeit.Int16)
f.Set("int32", gofakeit.Int32)
f.Set("int64", gofakeit.Int64)
f.Set("uint8", gofakeit.Uint8)
f.Set("uint16", gofakeit.Uint16)
f.Set("uint32", gofakeit.Uint32)
f.Set("uint64", gofakeit.Uint64)
f.Set("float32", gofakeit.Float32)
f.Set("float32_range", gofakeit.Float32Range)
f.Set("float64", gofakeit.Float64)
f.Set("float64_range", gofakeit.Float64Range)
f.Set("shuffle_ints", gofakeit.ShuffleInts)
f.Set("mac_address", gofakeit.MacAddress)
// String
f.Set("digit", gofakeit.Digit)
f.Set("letter", gofakeit.Letter)
f.Set("lexify", gofakeit.Lexify)
f.Set("rand_string", gofakeit.RandString)
f.Set("shuffle_strings", gofakeit.ShuffleStrings)
f.Set("numerify", gofakeit.Numerify)
//f.Set("programming_language", gofakeit.ProgrammingLanguage)
}

173
serv/serv.go
View File

@ -2,8 +2,6 @@ package serv
import (
"context"
"errors"
"flag"
"fmt"
"net/http"
"os"
@ -13,139 +11,8 @@ import (
"github.com/dosco/super-graph/psql"
"github.com/dosco/super-graph/qcode"
"github.com/go-pg/pg"
"github.com/gobuffalo/flect"
"github.com/rs/zerolog"
"github.com/spf13/viper"
)
//go:generate esc -o static.go -ignore \\.DS_Store -prefix ../web/build -private -pkg serv ../web/build
const (
serverName = "Super Graph"
authFailBlockAlways = iota + 1
authFailBlockPerQuery
authFailBlockNever
)
var (
logger *zerolog.Logger
conf *config
db *pg.DB
qcompile *qcode.Compiler
pcompile *psql.Compiler
authFailBlock int
)
func initLog() *zerolog.Logger {
logger := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).
With().
Timestamp().
Caller().
Logger()
return &logger
/*
log := logrus.New()
logger.Formatter = new(logrus.TextFormatter)
logger.Formatter.(*logrus.TextFormatter).DisableColors = false
logger.Formatter.(*logrus.TextFormatter).DisableTimestamp = true
logger.Level = logrus.TraceLevel
logger.Out = os.Stdout
*/
}
func initConf(path string) (*config, error) {
vi := viper.New()
vi.SetEnvPrefix("SG")
vi.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
vi.AutomaticEnv()
vi.AddConfigPath(path)
vi.AddConfigPath("./config")
vi.SetConfigName(getConfigName())
vi.SetDefault("host_port", "0.0.0.0:8080")
vi.SetDefault("web_ui", false)
vi.SetDefault("enable_tracing", false)
vi.SetDefault("auth_fail_block", "always")
vi.SetDefault("database.type", "postgres")
vi.SetDefault("database.host", "localhost")
vi.SetDefault("database.port", 5432)
vi.SetDefault("database.user", "postgres")
vi.SetDefault("env", "development")
vi.BindEnv("env", "GO_ENV")
vi.BindEnv("HOST", "HOST")
vi.BindEnv("PORT", "PORT")
vi.SetDefault("auth.rails.max_idle", 80)
vi.SetDefault("auth.rails.max_active", 12000)
if err := vi.ReadInConfig(); err != nil {
return nil, err
}
c := &config{}
if err := vi.Unmarshal(c); err != nil {
return nil, fmt.Errorf("unable to decode config, %v", err)
}
for k, v := range c.Inflections {
flect.AddPlural(k, v)
}
for i := range c.DB.Tables {
t := c.DB.Tables[i]
t.Name = flect.Pluralize(strings.ToLower(t.Name))
}
authFailBlock = getAuthFailBlock(c)
//fmt.Printf("%#v", c)
return c, nil
}
func initDB(c *config) (*pg.DB, error) {
opt := &pg.Options{
Addr: strings.Join([]string{c.DB.Host, c.DB.Port}, ":"),
User: c.DB.User,
Password: c.DB.Password,
Database: c.DB.DBName,
ApplicationName: c.AppName,
}
if c.DB.PoolSize != 0 {
opt.PoolSize = conf.DB.PoolSize
}
if c.DB.MaxRetries != 0 {
opt.MaxRetries = c.DB.MaxRetries
}
if len(c.DB.Schema) != 0 {
opt.OnConnect = func(conn *pg.Conn) error {
_, err := conn.Exec("set search_path=?", c.DB.Schema)
if err != nil {
return err
}
return nil
}
}
db := pg.Connect(opt)
if db == nil {
return nil, errors.New("failed to connect to postgres db")
}
return db, nil
}
func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
schema, err := psql.NewDBSchema(db, c.getAliasMap())
if err != nil {
@ -191,46 +58,6 @@ func initWatcher(path string) {
}()
}
func Init() {
var err error
path := flag.String("path", "./", "Path to config files")
flag.Parse()
logger = initLog()
conf, err = initConf(*path)
if err != nil {
logger.Fatal().Err(err).Msg("failed to read config")
}
logLevel, err := zerolog.ParseLevel(conf.LogLevel)
if err != nil {
logger.Error().Err(err).Msg("error setting log_level")
}
zerolog.SetGlobalLevel(logLevel)
db, err = initDB(conf)
if err != nil {
logger.Fatal().Err(err).Msg("failed to connect to database")
}
qcompile, pcompile, err = initCompilers(conf)
if err != nil {
logger.Fatal().Err(err).Msg("failed to connect to database")
}
if err := initResolvers(); err != nil {
logger.Fatal().Err(err).Msg("failed to initialized resolvers")
}
initAllowList(*path)
initPreparedList()
initWatcher(*path)
startHTTP()
}
func startHTTP() {
hp := strings.SplitN(conf.HostPort, ":", 2)