diff --git a/docker-compose.yml b/docker-compose.yml index 81f9290..2fcd94c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,15 +23,17 @@ services: command: fresh -c fresh.conf depends_on: - db - - rails_app rails_app: build: rails-app/. command: bash -c "rm -f tmp/pids/server.pid && bundle exec rails s -p 3000 -b '0.0.0.0'" volumes: - ./rails-app:/app + - /app/tmp ports: - "3000:3000" depends_on: - db + - super_graph + # - redis diff --git a/docs/guide.md b/docs/guide.md index 2b73a61..e372ef0 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -143,7 +143,7 @@ Postgres also supports full text search using a TSV index. Super Graph makes it ```graphql query { - products(search "amazing") { + products(search: "ale") { name } } @@ -371,6 +371,7 @@ tables: id: stripe_id url: http://rails_app:3000/stripe/$id path: data + # debug: true # pass_headers: # - cookie # - host @@ -417,8 +418,7 @@ And voila here is the result. You get all of this advanced and honestly complex ... ``` -Even tracing data is availble in the Super Graph web UI if tracing is enabled in the -config. By default it is for development. +Even tracing data is availble in the Super Graph web UI if tracing is enabled in the config. By default it is enabled in development. Additionally there you can set `debug: true` to enable http request / response dumping to help with debugging. ![Query Tracing](/tracing.png "Super Graph Web UI Query Tracing") diff --git a/go.mod b/go.mod index 7cc0b6b..96cadbf 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,9 @@ require ( github.com/garyburd/redigo v1.6.0 github.com/go-pg/pg v8.0.1+incompatible + github.com/gobuffalo/envy v1.7.0 // indirect github.com/gobuffalo/flect v0.1.1 + github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 github.com/gorilla/websocket v1.4.0 github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect github.com/labstack/gommon v0.2.8 diff --git a/go.sum b/go.sum index 38944cd..23aa80e 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,12 @@ github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/go-pg/pg v8.0.1+incompatible h1:gi93AxXmqlFGT0os5z2kTnbDqCk6BHXnA9MMApVxAkY= github.com/go-pg/pg v8.0.1+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/flect v0.1.1 h1:GTZJjJufv9FxgRs1+0Soo3wj+Md3kTUmTER/YE4uINA= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= @@ -39,8 +43,13 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0= github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= @@ -62,6 +71,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.1.0 h1:g0fH8RicVgNl+zVZDCDfbdWxAWoAEJyI7I3TZYXFiig= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0= github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg= @@ -116,6 +127,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= diff --git a/psql/bench.2 b/psql/bench.2 new file mode 100644 index 0000000..7a374d2 --- /dev/null +++ b/psql/bench.2 @@ -0,0 +1,7 @@ +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/psql +BenchmarkCompile-8 50000 26899 ns/op 4984 B/op 136 allocs/op +BenchmarkCompileParallel-8 200000 8128 ns/op 5046 B/op 136 allocs/op +PASS +ok github.com/dosco/super-graph/psql 3.455s diff --git a/psql/bench.3 b/psql/bench.3 new file mode 100644 index 0000000..d400c4f --- /dev/null +++ b/psql/bench.3 @@ -0,0 +1,7 @@ +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/psql +BenchmarkCompile-8 50000 25670 ns/op 4533 B/op 134 allocs/op +BenchmarkCompileParallel-8 200000 7533 ns/op 4590 B/op 134 allocs/op +PASS +ok github.com/dosco/super-graph/psql 3.149s diff --git a/psql/bench.4 b/psql/bench.4 new file mode 100644 index 0000000..a3e431f --- /dev/null +++ b/psql/bench.4 @@ -0,0 +1,7 @@ +goos: darwin +goarch: amd64 +pkg: github.com/dosco/super-graph/psql +BenchmarkCompile-8 100000 16476 ns/op 3282 B/op 66 allocs/op +BenchmarkCompileParallel-8 300000 4639 ns/op 3324 B/op 66 allocs/op +PASS +ok github.com/dosco/super-graph/psql 3.274s diff --git a/psql/psql.go b/psql/psql.go index 6c22ac6..e277d42 100644 --- a/psql/psql.go +++ b/psql/psql.go @@ -8,6 +8,7 @@ import ( "math" "strings" + "github.com/cespare/xxhash/v2" "github.com/dosco/super-graph/qcode" "github.com/dosco/super-graph/util" ) @@ -18,31 +19,30 @@ const ( ) type Config struct { - Schema *DBSchema - Vars map[string]string - TableMap map[string]string + Schema *DBSchema + Vars map[string]string } type Compiler struct { schema *DBSchema vars map[string]string - tmap map[string]string } func NewCompiler(conf Config) *Compiler { - return &Compiler{conf.Schema, conf.Vars, conf.TableMap} + return &Compiler{conf.Schema, conf.Vars} } -func (c *Compiler) AddRelationship(key uint64, val *DBRel) { - c.schema.RelMap[key] = val +func (c *Compiler) AddRelationship(child, parent string, rel *DBRel) error { + return c.schema.SetRel(child, parent, rel) } -func (c *Compiler) IDColumn(table string) string { - t, ok := c.schema.Tables[table] - if !ok { - return empty +func (c *Compiler) IDColumn(table string) (string, error) { + t, err := c.schema.GetTable(table) + if err != nil { + return empty, err } - return t.PrimaryCol + + return t.PrimaryCol, nil } type compilerContext struct { @@ -78,7 +78,6 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) { c.w.WriteString(`) FROM (`) var ignored uint32 - var err error for { if st.Len() == 0 { @@ -90,12 +89,17 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) { if id < closeBlock { sel := &c.s[id] + ti, err := c.schema.GetTable(sel.Table) + if err != nil { + return 0, err + } + if sel.ID != 0 { if err = c.renderJoin(sel); err != nil { return 0, err } } - skipped, err := c.renderSelect(sel) + skipped, err := c.renderSelect(sel, ti) if err != nil { return 0, err } @@ -113,7 +117,16 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) { } else { sel := &c.s[(id - closeBlock)] - err = c.renderSelectClose(sel) + + ti, err := c.schema.GetTable(sel.Table) + if err != nil { + return 0, err + } + + err = c.renderSelectClose(sel, ti) + if err != nil { + return 0, err + } if sel.ID != 0 { if err = c.renderJoinClose(sel); err != nil { @@ -130,16 +143,7 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) { return ignored, nil } -func (c *compilerContext) getTable(sel *qcode.Select) ( - *DBTableInfo, error) { - if tn, ok := c.tmap[sel.Table]; ok { - return c.schema.GetTable(tn) - } - - return c.schema.GetTable(sel.Table) -} - -func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.Column) { +func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column) { var skipped uint32 cols := make([]*qcode.Column, 0, len(sel.Cols)) @@ -152,8 +156,8 @@ func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.C for _, id := range sel.Children { child := &c.s[id] - rel, ok := c.schema.RelMap[child.RelID] - if !ok { + rel, err := c.schema.GetRel(child.Table, ti.Name) + if err != nil { skipped |= (1 << uint(id)) continue } @@ -183,12 +187,12 @@ func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.C return skipped, cols } -func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) { - skipped, childCols := c.processChildren(sel) +func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint32, error) { + skipped, childCols := c.processChildren(sel, ti) hasOrder := len(sel.OrderBy) != 0 // SELECT - if sel.AsList { + if ti.Singular == false { //fmt.Fprintf(w, `SELECT coalesce(json_agg("%s"`, c.sel.Table) c.w.WriteString(`SELECT coalesce(json_agg("`) c.w.WriteString(sel.Table) @@ -246,7 +250,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) { // END-SELECT // FROM (SELECT .... ) - err = c.renderBaseSelect(sel, childCols, skipped) + err = c.renderBaseSelect(sel, ti, childCols, skipped) if err != nil { return skipped, err } @@ -255,7 +259,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) { return skipped, nil } -func (c *compilerContext) renderSelectClose(sel *qcode.Select) error { +func (c *compilerContext) renderSelectClose(sel *qcode.Select, ti *DBTableInfo) error { hasOrder := len(sel.OrderBy) != 0 if hasOrder { @@ -270,6 +274,10 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select) error { c.w.WriteString(` LIMIT ('`) c.w.WriteString(sel.Paging.Limit) c.w.WriteString(`') :: integer`) + + } else if ti.Singular { + c.w.WriteString(` LIMIT ('1') :: integer`) + } else { c.w.WriteString(` LIMIT ('20') :: integer`) } @@ -281,7 +289,7 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select) error { c.w.WriteString(`') :: integer`) } - if sel.AsList { + if ti.Singular == false { //fmt.Fprintf(w, `) AS "%s_%d"`, c.sel.Table, c.sel.ID) c.w.WriteString(`)`) aliasWithID(c.w, sel.Table, sel.ID) @@ -304,16 +312,21 @@ func (c *compilerContext) renderJoinClose(sel *qcode.Select) error { } func (c *compilerContext) renderJoinTable(sel *qcode.Select) { - rel, ok := c.schema.RelMap[sel.RelID] - if !ok { - panic(errors.New("no relationship found")) + parent := &c.s[sel.ParentID] + + rel, err := c.schema.GetRel(sel.Table, parent.Table) + if err != nil { + panic(err) } if rel.Type != RelOneToManyThrough { return } - parent := &c.s[sel.ParentID] + pt, err := c.schema.GetTable(parent.Table) + if err != nil { + return + } //fmt.Fprintf(w, ` LEFT OUTER JOIN "%s" ON (("%s"."%s") = ("%s_%d"."%s"))`, //rel.Through, rel.Through, rel.ColT, c.parent.Table, c.parent.ID, rel.Col1) @@ -322,7 +335,7 @@ func (c *compilerContext) renderJoinTable(sel *qcode.Select) { c.w.WriteString(`" ON ((`) colWithTable(c.w, rel.Through, rel.ColT) c.w.WriteString(`) = (`) - colWithTableID(c.w, parent.Table, parent.ID, rel.Col1) + colWithTableID(c.w, pt.Name, parent.ID, rel.Col1) c.w.WriteString(`))`) } @@ -343,8 +356,8 @@ func (c *compilerContext) renderRemoteRelColumns(sel *qcode.Select) { for _, id := range sel.Children { child := &c.s[id] - rel, ok := c.schema.RelMap[child.RelID] - if !ok || rel.Type != RelRemote { + rel, err := c.schema.GetRel(child.Table, sel.Table) + if err != nil || rel.Type != RelRemote { continue } if i != 0 || len(sel.Cols) != 0 { @@ -380,15 +393,10 @@ func (c *compilerContext) renderJoinedColumns(sel *qcode.Select, skipped uint32) return nil } -func (c *compilerContext) renderBaseSelect(sel *qcode.Select, +func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, childCols []*qcode.Column, skipped uint32) error { var groupBy []int - ti, err := c.getTable(sel) - if err != nil { - return err - } - isRoot := sel.ID == 0 isFil := sel.Where != nil isSearch := sel.Args["search"] != nil @@ -473,19 +481,30 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, } c.w.WriteString(` FROM `) - if tn, ok := c.tmap[sel.Table]; ok { + + if c.schema.IsAlias(sel.Table) || ti.Singular { //fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Table) - colWithAlias(c.w, tn, sel.Table) + tableWithAlias(c.w, ti.Name, sel.Table) } else { //fmt.Fprintf(w, ` FROM "%s"`, c.sel.Table) c.w.WriteString(`"`) - c.w.WriteString(sel.Table) + c.w.WriteString(ti.Name) c.w.WriteString(`"`) } + // if tn, ok := c.tmap[sel.Table]; ok { + // //fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Table) + // tableWithAlias(c.w, ti.Name, sel.Table) + // } else { + // //fmt.Fprintf(w, ` FROM "%s"`, c.sel.Table) + // c.w.WriteString(`"`) + // c.w.WriteString(sel.Table) + // c.w.WriteString(`"`) + // } + if isRoot && isFil { c.w.WriteString(` WHERE (`) - if err := c.renderWhere(sel); err != nil { + if err := c.renderWhere(sel, ti); err != nil { return err } c.w.WriteString(`)`) @@ -499,7 +518,7 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, if isFil { c.w.WriteString(` AND `) - if err := c.renderWhere(sel); err != nil { + if err := c.renderWhere(sel, ti); err != nil { return err } } @@ -525,6 +544,10 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, c.w.WriteString(` LIMIT ('`) c.w.WriteString(sel.Paging.Limit) c.w.WriteString(`') :: integer`) + + } else if ti.Singular { + c.w.WriteString(` LIMIT ('1') :: integer`) + } else { c.w.WriteString(` LIMIT ('20') :: integer`) } @@ -562,13 +585,13 @@ func (c *compilerContext) renderOrderByColumns(sel *qcode.Select) { } func (c *compilerContext) renderRelationship(sel *qcode.Select) { - rel, ok := c.schema.RelMap[sel.RelID] - if !ok { - panic(errors.New("no relationship found")) - } - parent := c.s[sel.ParentID] + rel, err := c.schema.GetRel(sel.Table, parent.Table) + if err != nil { + panic(err) + } + switch rel.Type { case RelBelongTo: //fmt.Fprintf(w, `(("%s"."%s") = ("%s_%d"."%s"))`, @@ -599,18 +622,13 @@ func (c *compilerContext) renderRelationship(sel *qcode.Select) { } } -func (c *compilerContext) renderWhere(sel *qcode.Select) error { +func (c *compilerContext) renderWhere(sel *qcode.Select, ti *DBTableInfo) error { st := util.NewStack() if sel.Where != nil { st.Push(sel.Where) } - ti, err := c.getTable(sel) - if err != nil { - return err - } - for { if st.Len() == 0 { break @@ -909,6 +927,14 @@ func colWithAlias(w *bytes.Buffer, col, alias string) { w.WriteString(`"`) } +func tableWithAlias(w *bytes.Buffer, table, alias string) { + w.WriteString(`"`) + w.WriteString(table) + w.WriteString(`" AS "`) + w.WriteString(alias) + w.WriteString(`"`) +} + func colWithTable(w *bytes.Buffer, table, col string) { w.WriteString(`"`) w.WriteString(table) @@ -987,3 +1013,11 @@ func int2string(w *bytes.Buffer, val int32) { w.WriteByte(charset[d]) } } + +func relID(h *xxhash.Digest, child, parent string) uint64 { + h.WriteString(child) + h.WriteString(parent) + v := h.Sum64() + h.Reset() + return v +} diff --git a/psql/psql_test.go b/psql/psql_test.go index 0c5fb3d..6d2bef4 100644 --- a/psql/psql_test.go +++ b/psql/psql_test.go @@ -100,12 +100,17 @@ func TestMain(m *testing.M) { } schema := &DBSchema{ - Tables: make(map[string]*DBTableInfo), - RelMap: make(map[uint64]*DBRel), + t: make(map[string]*DBTableInfo), + rm: make(map[string]map[string]*DBRel), + al: make(map[string]struct{}), + } + + aliases := map[string][]string{ + "users": []string{"mes"}, } for i, t := range tables { - schema.updateSchema(t, columns[i]) + schema.updateSchema(t, columns[i], aliases) } vars := NewVariables(map[string]string{ @@ -115,9 +120,6 @@ func TestMain(m *testing.M) { pcompile = NewCompiler(Config{ Schema: schema, Vars: vars, - TableMap: map[string]string{ - "mes": "users", - }, }) os.Exit(m.Run()) @@ -260,7 +262,7 @@ func fetchByID(t *testing.T) { } }` - sql := `SELECT json_object_agg('product', products) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name") AS "sel_0")) AS "products" FROM (SELECT "products"."id", "products"."name" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8)) AND (("id") = (15))) LIMIT ('1') :: integer) AS "products_0" LIMIT ('1') :: integer) AS "done_1337";` + sql := `SELECT json_object_agg('product', product) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "product_0"."id" AS "id", "product_0"."name" AS "name") AS "sel_0")) AS "product" FROM (SELECT "product"."id", "product"."name" FROM "products" AS "product" WHERE ((("product"."price") > (0)) AND (("product"."price") < (8)) AND (("id") = (15))) LIMIT ('1') :: integer) AS "product_0" LIMIT ('1') :: integer) AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { @@ -432,7 +434,7 @@ func queryWithVariables(t *testing.T) { } }` - sql := `SELECT json_object_agg('product', products) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name") AS "sel_0")) AS "products" FROM (SELECT "products"."id", "products"."name" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8)) AND (("products"."price") = ('{{PRODUCT_PRICE}}')) AND (("id") = ('{{PRODUCT_ID}}'))) LIMIT ('1') :: integer) AS "products_0" LIMIT ('1') :: integer) AS "done_1337";` + sql := `SELECT json_object_agg('product', product) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "product_0"."id" AS "id", "product_0"."name" AS "name") AS "sel_0")) AS "product" FROM (SELECT "product"."id", "product"."name" FROM "products" AS "product" WHERE ((("product"."price") > (0)) AND (("product"."price") < (8)) AND (("product"."price") = ('{{PRODUCT_PRICE}}')) AND (("id") = ('{{PRODUCT_ID}}'))) LIMIT ('1') :: integer) AS "product_0" LIMIT ('1') :: integer) AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { @@ -451,7 +453,7 @@ func syntheticTables(t *testing.T) { } }` - sql := `SELECT json_object_agg('me', mes) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "mes_0"."email" AS "email") AS "sel_0")) AS "mes" FROM (SELECT "mes"."email" FROM "users" AS "mes" WHERE ((("mes"."id") = ('{{user_id}}'))) LIMIT ('1') :: integer) AS "mes_0" LIMIT ('1') :: integer) AS "done_1337";` + sql := `SELECT json_object_agg('me', me) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "me_0"."email" AS "email") AS "sel_0")) AS "me" FROM (SELECT "me"."email" FROM "users" AS "me" WHERE ((("me"."id") = ('{{user_id}}'))) LIMIT ('1') :: integer) AS "me_0" LIMIT ('1') :: integer) AS "done_1337";` resSQL, err := compileGQLToPSQL(gql) if err != nil { diff --git a/psql/tables.go b/psql/tables.go index 80839b8..84e11d5 100644 --- a/psql/tables.go +++ b/psql/tables.go @@ -4,166 +4,10 @@ import ( "fmt" "strings" - "github.com/cespare/xxhash/v2" "github.com/go-pg/pg" + "github.com/gobuffalo/flect" ) -type DBSchema struct { - Tables map[string]*DBTableInfo - RelMap map[uint64]*DBRel -} - -type DBTableInfo struct { - Name string - PrimaryCol string - TSVCol string - Columns map[string]*DBColumn -} - -type RelType int - -const ( - RelBelongTo RelType = iota + 1 - RelOneToMany - RelOneToManyThrough - RelRemote -) - -type DBRel struct { - Type RelType - Through string - ColT string - Col1 string - Col2 string -} - -func NewDBSchema(db *pg.DB) (*DBSchema, error) { - schema := &DBSchema{ - Tables: make(map[string]*DBTableInfo), - RelMap: make(map[uint64]*DBRel), - } - - tables, err := GetTables(db) - if err != nil { - return nil, err - } - - for _, t := range tables { - cols, err := GetColumns(db, "public", t.Name) - if err != nil { - return nil, err - } - - schema.updateSchema(t, cols) - } - - return schema, nil -} - -func (s *DBSchema) updateSchema(t *DBTable, cols []*DBColumn) { - // Current table - ti := &DBTableInfo{ - Name: t.Name, - Columns: make(map[string]*DBColumn, len(cols)), - } - - // Foreign key columns in current table - var jcols []*DBColumn - colByID := make(map[int]*DBColumn) - - for i := range cols { - c := cols[i] - ti.Columns[strings.ToLower(c.Name)] = cols[i] - colByID[c.ID] = cols[i] - } - - ct := strings.ToLower(t.Name) - s.Tables[ct] = ti - - h := xxhash.New() - - for _, c := range cols { - switch { - case c.Type == "tsvector": - s.Tables[ct].TSVCol = c.Name - - case c.PrimaryKey: - s.Tables[ct].PrimaryCol = c.Name - - case len(c.FKeyTable) != 0: - if len(c.FKeyColID) == 0 { - continue - } - - // Foreign key column name - ft := strings.ToLower(c.FKeyTable) - fc, ok := colByID[c.FKeyColID[0]] - if !ok { - continue - } - - // Belongs-to relation between current table and the - // table in the foreign key - rel1 := &DBRel{RelBelongTo, "", "", c.Name, fc.Name} - s.RelMap[relID(h, ct, ft)] = rel1 - - // One-to-many relation between the foreign key table and the - // the current table - rel2 := &DBRel{RelOneToMany, "", "", fc.Name, c.Name} - s.RelMap[relID(h, ft, ct)] = rel2 - - jcols = append(jcols, c) - } - } - - // If table contains multiple foreign key columns it's a possible - // join table for many-to-many relationships or multiple one-to-many - // relations - - // Below one-to-many relations use the current table as the - // join table aka through table. - if len(jcols) > 1 { - for i := range jcols { - for n := range jcols { - if n != i { - s.updateSchemaOTMT(h, ct, jcols[i], jcols[n], colByID) - } - } - } - } -} - -func (s *DBSchema) updateSchemaOTMT( - h *xxhash.Digest, - ct string, - col1, col2 *DBColumn, - colByID map[int]*DBColumn) { - - t1 := strings.ToLower(col1.FKeyTable) - t2 := strings.ToLower(col2.FKeyTable) - - fc1, ok := colByID[col1.FKeyColID[0]] - if !ok { - return - } - fc2, ok := colByID[col2.FKeyColID[0]] - if !ok { - return - } - - // One-to-many-through relation between 1nd foreign key table and the - // 2nd foreign key table - //rel1 := &DBRel{RelOneToManyThrough, ct, fc1.Name, col1.Name} - rel1 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name, col1.Name} - s.RelMap[relID(h, t1, t2)] = rel1 - - // One-to-many-through relation between 2nd foreign key table and the - // 1nd foreign key table - //rel2 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name} - rel2 := &DBRel{RelOneToManyThrough, ct, col1.Name, fc1.Name, col2.Name} - s.RelMap[relID(h, t2, t1)] = rel2 -} - type DBTable struct { Name string `sql:"name"` Type string `sql:"type"` @@ -255,18 +99,231 @@ WHERE c.relkind = 'r'::char return t, nil } +type DBSchema struct { + t map[string]*DBTableInfo + rm map[string]map[string]*DBRel + al map[string]struct{} +} + +type DBTableInfo struct { + Name string + Singular bool + PrimaryCol string + TSVCol string + Columns map[string]*DBColumn +} + +type RelType int + +const ( + RelBelongTo RelType = iota + 1 + RelOneToMany + RelOneToManyThrough + RelRemote +) + +type DBRel struct { + Type RelType + Through string + ColT string + Col1 string + Col2 string +} + +func NewDBSchema(db *pg.DB, aliases map[string][]string) (*DBSchema, error) { + schema := &DBSchema{ + t: make(map[string]*DBTableInfo), + rm: make(map[string]map[string]*DBRel), + al: make(map[string]struct{}), + } + + tables, err := GetTables(db) + if err != nil { + return nil, err + } + + for _, t := range tables { + cols, err := GetColumns(db, "public", t.Name) + if err != nil { + return nil, err + } + + schema.updateSchema(t, cols, aliases) + } + + return schema, nil +} + +func (s *DBSchema) updateSchema( + t *DBTable, + cols []*DBColumn, + aliases map[string][]string) { + + // Foreign key columns in current table + colByID := make(map[int]*DBColumn) + columns := make(map[string]*DBColumn, len(cols)) + + for i := range cols { + c := cols[i] + columns[strings.ToLower(c.Name)] = cols[i] + colByID[c.ID] = cols[i] + } + + singular := strings.ToLower(flect.Singularize(t.Name)) + s.t[singular] = &DBTableInfo{ + Name: t.Name, + Singular: true, + Columns: columns, + } + + plural := strings.ToLower(flect.Pluralize(t.Name)) + s.t[plural] = &DBTableInfo{ + Name: t.Name, + Singular: false, + Columns: columns, + } + + ct := strings.ToLower(t.Name) + + if al, ok := aliases[ct]; ok { + for i := range al { + k1 := flect.Singularize(al[i]) + s.t[k1] = s.t[singular] + + k2 := flect.Pluralize(al[i]) + s.t[k2] = s.t[plural] + + s.al[k1] = struct{}{} + s.al[k2] = struct{}{} + } + } + + jcols := make([]*DBColumn, 0, len(cols)) + + for _, c := range cols { + switch { + case c.Type == "tsvector": + s.t[singular].TSVCol = c.Name + s.t[plural].TSVCol = c.Name + + case c.PrimaryKey: + s.t[singular].PrimaryCol = c.Name + s.t[plural].PrimaryCol = c.Name + + case len(c.FKeyTable) != 0: + if len(c.FKeyColID) == 0 { + continue + } + + // Foreign key column name + ft := strings.ToLower(c.FKeyTable) + fc, ok := colByID[c.FKeyColID[0]] + if !ok { + continue + } + + // Belongs-to relation between current table and the + // table in the foreign key + rel1 := &DBRel{RelBelongTo, "", "", c.Name, fc.Name} + s.SetRel(ct, ft, rel1) + + // One-to-many relation between the foreign key table and the + // the current table + rel2 := &DBRel{RelOneToMany, "", "", fc.Name, c.Name} + s.SetRel(ft, ct, rel2) + + jcols = append(jcols, c) + } + } + + // If table contains multiple foreign key columns it's a possible + // join table for many-to-many relationships or multiple one-to-many + // relations + + // Below one-to-many relations use the current table as the + // join table aka through table. + if len(jcols) > 1 { + for i := range jcols { + for n := range jcols { + if n != i { + s.updateSchemaOTMT(ct, jcols[i], jcols[n], colByID) + } + } + } + } +} + +func (s *DBSchema) updateSchemaOTMT( + ct string, + col1, col2 *DBColumn, + colByID map[int]*DBColumn) { + + t1 := strings.ToLower(col1.FKeyTable) + t2 := strings.ToLower(col2.FKeyTable) + + fc1, ok := colByID[col1.FKeyColID[0]] + if !ok { + return + } + fc2, ok := colByID[col2.FKeyColID[0]] + if !ok { + return + } + + // One-to-many-through relation between 1nd foreign key table and the + // 2nd foreign key table + //rel1 := &DBRel{RelOneToManyThrough, ct, fc1.Name, col1.Name} + rel1 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name, col1.Name} + s.SetRel(t1, t2, rel1) + + // One-to-many-through relation between 2nd foreign key table and the + // 1nd foreign key table + //rel2 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name} + rel2 := &DBRel{RelOneToManyThrough, ct, col1.Name, fc1.Name, col2.Name} + s.SetRel(t2, t1, rel2) +} + func (s *DBSchema) GetTable(table string) (*DBTableInfo, error) { - t, ok := s.Tables[table] + t, ok := s.t[table] if !ok { return nil, fmt.Errorf("unknown table '%s'", table) } return t, nil } -func relID(h *xxhash.Digest, child, parent string) uint64 { - h.WriteString(child) - h.WriteString(parent) - v := h.Sum64() - h.Reset() - return v +func (s *DBSchema) SetRel(child, parent string, rel *DBRel) error { + sc := strings.ToLower(flect.Singularize(child)) + pc := strings.ToLower(flect.Pluralize(child)) + + if _, ok := s.rm[sc]; !ok { + s.rm[sc] = make(map[string]*DBRel) + } + + if _, ok := s.rm[pc]; !ok { + s.rm[pc] = make(map[string]*DBRel) + } + + sp := strings.ToLower(flect.Singularize(parent)) + pp := strings.ToLower(flect.Pluralize(parent)) + + s.rm[sc][sp] = rel + s.rm[sc][pp] = rel + s.rm[pc][sp] = rel + s.rm[pc][pp] = rel + + return nil +} + +func (s *DBSchema) GetRel(child, parent string) (*DBRel, error) { + rel, ok := s.rm[child][parent] + if !ok { + return nil, fmt.Errorf("unknown relationship '%s' -> '%s'", + child, parent) + } + return rel, nil +} + +func (s *DBSchema) IsAlias(name string) bool { + _, ok := s.al[name] + return ok } diff --git a/qcode/lex.go b/qcode/lex.go index bde5f1a..25ed9e8 100644 --- a/qcode/lex.go +++ b/qcode/lex.go @@ -303,6 +303,8 @@ func lexName(l *lexer) stateFn { l.backup() v := l.current() + lowercase(l.input, s, e) + if len(v) == 0 { switch { case strings.EqualFold(v, "query"): diff --git a/qcode/qcode.go b/qcode/qcode.go index f823bea..01eb6ab 100644 --- a/qcode/qcode.go +++ b/qcode/qcode.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/cespare/xxhash/v2" "github.com/dosco/super-graph/util" "github.com/gobuffalo/flect" ) @@ -31,11 +30,8 @@ type Column struct { type Select struct { ID int32 ParentID int32 - RelID uint64 Args map[string]*Node - AsList bool Table string - Singular string FieldName string Cols []Column Where *Exp @@ -163,7 +159,12 @@ func NewCompiler(c Config) (*Compiler, error) { if err != nil { return nil, err } - fm[strings.ToLower(k)] = fil + k1 := strings.ToLower(k) + singular := flect.Singularize(k1) + plural := flect.Pluralize(k1) + + fm[singular] = fil + fm[plural] = fil } return &Compiler{fl, fm, bl, c.KeepArgs}, nil @@ -202,7 +203,6 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) { selects := make([]Select, 0, 5) st := util.NewStack() - h := xxhash.New() if len(op.Fields) == 0 { return nil, errors.New("empty query") @@ -226,46 +226,31 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) { } field := &op.Fields[fid] - fn := strings.ToLower(field.Name) - if _, ok := com.bl[fn]; ok { + tn := strings.ToLower(field.Name) + if _, ok := com.bl[tn]; ok { continue } - tn := flect.Pluralize(fn) - s := Select{ + selects = append(selects, Select{ ID: id, ParentID: parentID, Table: tn, Children: make([]int32, 0, 5), - } + }) + s := &selects[(len(selects) - 1)] if s.ID != 0 { p := &selects[s.ParentID] p.Children = append(p.Children, s.ID) - s.RelID = relID(h, tn, p.Table) - } - - if fn == tn { - s.Singular = flect.Singularize(fn) - } else { - s.Singular = fn - } - - if fn == s.Table { - s.AsList = true - } else { - s.Paging.Limit = "1" } if len(field.Alias) != 0 { s.FieldName = field.Alias - } else if s.AsList { - s.FieldName = s.Table } else { - s.FieldName = s.Singular + s.FieldName = s.Table } - err := com.compileArgs(&s, field.Args) + err := com.compileArgs(s, field.Args) if err != nil { return nil, err } @@ -296,7 +281,6 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) { s.Cols = append(s.Cols, col) } - selects = append(selects, s) id++ } @@ -858,14 +842,6 @@ func buildPath(a []string) string { return b.String() } -func relID(h *xxhash.Digest, child, parent string) uint64 { - h.WriteString(child) - h.WriteString(parent) - v := h.Sum64() - h.Reset() - return v -} - func (t ExpOp) String() string { var v string diff --git a/rails-app/demo.yml b/rails-app/demo.yml index d8baa01..d474962 100644 --- a/rails-app/demo.yml +++ b/rails-app/demo.yml @@ -13,7 +13,7 @@ services: ports: - "8080:8080" - web: + rails_app: image: dosco/super-graph-demo:latest environment: RAILS_ENV: "development" diff --git a/serv/config.go b/serv/config.go index 3902ea1..198690c 100644 --- a/serv/config.go +++ b/serv/config.go @@ -1,6 +1,8 @@ package serv import ( + "strings" + "github.com/gobuffalo/flect" ) @@ -84,8 +86,8 @@ type configRemote struct { } `mapstructure:"set_headers"` } -func (c *config) getAliasMap() map[string]string { - m := make(map[string]string, len(c.DB.Tables)) +func (c *config) getAliasMap() map[string][]string { + m := make(map[string][]string, len(c.DB.Tables)) for i := range c.DB.Tables { t := c.DB.Tables[i] @@ -93,7 +95,9 @@ func (c *config) getAliasMap() map[string]string { if len(t.Table) == 0 { continue } - m[flect.Pluralize(t.Name)] = t.Table + + k := strings.ToLower(t.Table) + m[k] = append(m[k], strings.ToLower(t.Name)) } return m } @@ -107,12 +111,16 @@ func (c *config) getFilterMap() map[string][]string { if len(t.Filter) == 0 { continue } - name := flect.Pluralize(t.Name) + singular := flect.Singularize(t.Name) + plural := flect.Pluralize(t.Name) if t.Filter[0] == "none" { - m[name] = []string{} + m[singular] = []string{} + m[plural] = []string{} + } else { - m[name] = t.Filter + m[singular] = t.Filter + m[plural] = t.Filter } } diff --git a/serv/core.go b/serv/core.go index a1303c1..c069902 100644 --- a/serv/core.go +++ b/serv/core.go @@ -231,7 +231,6 @@ func (c *coreContext) resolveRemotes( to[n] = jsn.Field{[]byte(s.FieldName), ob.Bytes()} }(i, id, s) - fmt.Println(">>>", i) } wg.Wait() diff --git a/serv/reso.go b/serv/reso.go index 7775142..2706447 100644 --- a/serv/reso.go +++ b/serv/reso.go @@ -22,16 +22,21 @@ type resolvFn struct { Fn func(r *http.Request, id []byte) ([]byte, error) } -func initResolvers() { +func initResolvers() error { rmap = make(map[uint64]*resolvFn) for _, t := range conf.DB.Tables { - initRemotes(t) + err := initRemotes(t) + if err != nil { + return err + } } + return nil } -func initRemotes(t configTable) { +func initRemotes(t configTable) error { h := xxhash.New() + var err error for _, r := range t.Remotes { // defines the table column to be used as an id in the @@ -41,24 +46,26 @@ func initRemotes(t configTable) { // if no table column specified in the config then // use the primary key of the table as the id if len(idcol) == 0 { - idcol = pcompile.IDColumn(t.Name) + idcol, err = pcompile.IDColumn(t.Name) + if err != nil { + return err + } } idk := fmt.Sprintf("__%s_%s", t.Name, idcol) // register a relationship between the remote data // and the database table - h.WriteString(strings.ToLower(r.Name)) - h.WriteString(t.Name) - key := h.Sum64() - h.Reset() - val := &psql.DBRel{ Type: psql.RelRemote, Col1: idcol, Col2: idk, } - pcompile.AddRelationship(key, val) + + err := pcompile.AddRelationship(strings.ToLower(r.Name), t.Name, val) + if err != nil { + return err + } // the function thats called to resolve this remote // data request @@ -81,6 +88,8 @@ func initRemotes(t configTable) { // index resolver obj by IDField rmap[xxhash.Sum64(rf.IDField)] = rf } + + return nil } func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { @@ -88,7 +97,8 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { client := &http.Client{} fn := func(inReq *http.Request, id []byte) ([]byte, error) { - req, err := http.NewRequest("GET", fmt.Sprintf(reqURL, id), nil) + uri := fmt.Sprintf(reqURL, id) + req, err := http.NewRequest("GET", uri, nil) if err != nil { return nil, err } @@ -107,6 +117,7 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) { res, err := client.Do(req) if err != nil { + logger.Error().Err(err).Msgf("Failed to connect to: %s", uri) return nil, err } defer res.Body.Close() diff --git a/serv/serv.go b/serv/serv.go index f966987..3ddc8c7 100644 --- a/serv/serv.go +++ b/serv/serv.go @@ -150,7 +150,7 @@ func initDB(c *config) (*pg.DB, error) { } func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) { - schema, err := psql.NewDBSchema(db) + schema, err := psql.NewDBSchema(db, c.getAliasMap()) if err != nil { return nil, nil, err } @@ -167,9 +167,8 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) { } pc := psql.NewCompiler(psql.Config{ - Schema: schema, - Vars: c.DB.Variables, - TableMap: c.getAliasMap(), + Schema: schema, + Vars: c.DB.Variables, }) return qc, pc, nil @@ -182,20 +181,22 @@ func Init() { conf, err = initConf() if err != nil { - logger.Fatal().Err(err) + logger.Fatal().Err(err).Msg("failed to read config") } db, err = initDB(conf) if err != nil { - logger.Fatal().Err(err) + logger.Fatal().Err(err).Msg("failed to connect to database") } qcompile, pcompile, err = initCompilers(conf) if err != nil { - logger.Fatal().Err(err) + logger.Fatal().Err(err).Msg("failed to connect to database") } - initResolvers() + if err := initResolvers(); err != nil { + logger.Fatal().Err(err).Msg("failed to initialized resolvers") + } startHTTP() } @@ -216,21 +217,21 @@ func startHTTP() { <-sigint if err := srv.Shutdown(context.Background()); err != nil { - logger.Printf("http: %v", err) + logger.Error().Err(err).Msg("shutdown signal received") } close(idleConnsClosed) }() srv.RegisterOnShutdown(func() { if err := db.Close(); err != nil { - logger.Error().Err(err) + logger.Error().Err(err).Msg("db closed") } }) fmt.Printf("%s listening on %s (%s)\n", serverName, conf.HostPort, conf.Env) if err := srv.ListenAndServe(); err != http.ErrServerClosed { - fmt.Println(err) + logger.Error().Err(err).Msg("server closed") } <-idleConnsClosed