Futher reduce allocations on the compiler hot path

This commit is contained in:
Vikram Rangnekar 2019-06-14 00:32:15 -04:00
parent 9aa4928d7b
commit 9af320f396
17 changed files with 434 additions and 306 deletions

View File

@ -23,15 +23,17 @@ services:
command: fresh -c fresh.conf
depends_on:
- db
- rails_app
rails_app:
build: rails-app/.
command: bash -c "rm -f tmp/pids/server.pid && bundle exec rails s -p 3000 -b '0.0.0.0'"
volumes:
- ./rails-app:/app
- /app/tmp
ports:
- "3000:3000"
depends_on:
- db
- super_graph
# - redis

View File

@ -143,7 +143,7 @@ Postgres also supports full text search using a TSV index. Super Graph makes it
```graphql
query {
products(search "amazing") {
products(search: "ale") {
name
}
}
@ -371,6 +371,7 @@ tables:
id: stripe_id
url: http://rails_app:3000/stripe/$id
path: data
# debug: true
# pass_headers:
# - cookie
# - host
@ -417,8 +418,7 @@ And voila here is the result. You get all of this advanced and honestly complex
...
```
Even tracing data is availble in the Super Graph web UI if tracing is enabled in the
config. By default it is for development.
Even tracing data is availble in the Super Graph web UI if tracing is enabled in the config. By default it is enabled in development. Additionally there you can set `debug: true` to enable http request / response dumping to help with debugging.
![Query Tracing](/tracing.png "Super Graph Web UI Query Tracing")

2
go.mod
View File

@ -11,7 +11,9 @@ require (
github.com/garyburd/redigo v1.6.0
github.com/go-pg/pg v8.0.1+incompatible
github.com/gobuffalo/envy v1.7.0 // indirect
github.com/gobuffalo/flect v0.1.1
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2
github.com/gorilla/websocket v1.4.0
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a // indirect
github.com/labstack/gommon v0.2.8

13
go.sum
View File

@ -27,8 +27,12 @@ github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc
github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/go-pg/pg v8.0.1+incompatible h1:gi93AxXmqlFGT0os5z2kTnbDqCk6BHXnA9MMApVxAkY=
github.com/go-pg/pg v8.0.1+incompatible/go.mod h1:a2oXow+aFOrvwcKs3eIA0lNFmMilrxK2sOkB5NWe0vA=
github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU=
github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
github.com/gobuffalo/flect v0.1.1 h1:GTZJjJufv9FxgRs1+0Soo3wj+Md3kTUmTER/YE4uINA=
github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI=
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2 h1:8thhT+kUJMTMy3HlX4+y9Da+BNJck+p109tqqKp7WDs=
github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
@ -39,8 +43,13 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a h1:eeaG9XMUvRBYXJi4pg1ZKM7nxc5AfXfojeLLW7O5J3k=
github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/labstack/gommon v0.2.8 h1:JvRqmeZcfrHC5u6uVleB4NxxNbzx6gpbJiQknDbKQu0=
github.com/labstack/gommon v0.2.8/go.mod h1:/tj9csK2iPSBvn+3NLM9e52usepMtrd5ilFYA+wQNJ4=
github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
@ -62,6 +71,8 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.1.0 h1:g0fH8RicVgNl+zVZDCDfbdWxAWoAEJyI7I3TZYXFiig=
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/zerolog v1.14.3 h1:4EGfSkR2hJDB0s3oFfrlPqjU1e4WLncergLil3nEKW0=
github.com/rs/zerolog v1.14.3/go.mod h1:3WXPzbXEEliJ+a6UFE4vhIxV8qR1EML6ngzP9ug4eYg=
@ -116,6 +127,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=

7
psql/bench.2 Normal file
View File

@ -0,0 +1,7 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompile-8 50000 26899 ns/op 4984 B/op 136 allocs/op
BenchmarkCompileParallel-8 200000 8128 ns/op 5046 B/op 136 allocs/op
PASS
ok github.com/dosco/super-graph/psql 3.455s

7
psql/bench.3 Normal file
View File

@ -0,0 +1,7 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompile-8 50000 25670 ns/op 4533 B/op 134 allocs/op
BenchmarkCompileParallel-8 200000 7533 ns/op 4590 B/op 134 allocs/op
PASS
ok github.com/dosco/super-graph/psql 3.149s

7
psql/bench.4 Normal file
View File

@ -0,0 +1,7 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/psql
BenchmarkCompile-8 100000 16476 ns/op 3282 B/op 66 allocs/op
BenchmarkCompileParallel-8 300000 4639 ns/op 3324 B/op 66 allocs/op
PASS
ok github.com/dosco/super-graph/psql 3.274s

View File

@ -8,6 +8,7 @@ import (
"math"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/dosco/super-graph/qcode"
"github.com/dosco/super-graph/util"
)
@ -20,29 +21,28 @@ const (
type Config struct {
Schema *DBSchema
Vars map[string]string
TableMap map[string]string
}
type Compiler struct {
schema *DBSchema
vars map[string]string
tmap map[string]string
}
func NewCompiler(conf Config) *Compiler {
return &Compiler{conf.Schema, conf.Vars, conf.TableMap}
return &Compiler{conf.Schema, conf.Vars}
}
func (c *Compiler) AddRelationship(key uint64, val *DBRel) {
c.schema.RelMap[key] = val
func (c *Compiler) AddRelationship(child, parent string, rel *DBRel) error {
return c.schema.SetRel(child, parent, rel)
}
func (c *Compiler) IDColumn(table string) string {
t, ok := c.schema.Tables[table]
if !ok {
return empty
func (c *Compiler) IDColumn(table string) (string, error) {
t, err := c.schema.GetTable(table)
if err != nil {
return empty, err
}
return t.PrimaryCol
return t.PrimaryCol, nil
}
type compilerContext struct {
@ -78,7 +78,6 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) {
c.w.WriteString(`) FROM (`)
var ignored uint32
var err error
for {
if st.Len() == 0 {
@ -90,12 +89,17 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) {
if id < closeBlock {
sel := &c.s[id]
ti, err := c.schema.GetTable(sel.Table)
if err != nil {
return 0, err
}
if sel.ID != 0 {
if err = c.renderJoin(sel); err != nil {
return 0, err
}
}
skipped, err := c.renderSelect(sel)
skipped, err := c.renderSelect(sel, ti)
if err != nil {
return 0, err
}
@ -113,7 +117,16 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) {
} else {
sel := &c.s[(id - closeBlock)]
err = c.renderSelectClose(sel)
ti, err := c.schema.GetTable(sel.Table)
if err != nil {
return 0, err
}
err = c.renderSelectClose(sel, ti)
if err != nil {
return 0, err
}
if sel.ID != 0 {
if err = c.renderJoinClose(sel); err != nil {
@ -130,16 +143,7 @@ func (co *Compiler) Compile(qc *qcode.QCode, w *bytes.Buffer) (uint32, error) {
return ignored, nil
}
func (c *compilerContext) getTable(sel *qcode.Select) (
*DBTableInfo, error) {
if tn, ok := c.tmap[sel.Table]; ok {
return c.schema.GetTable(tn)
}
return c.schema.GetTable(sel.Table)
}
func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.Column) {
func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column) {
var skipped uint32
cols := make([]*qcode.Column, 0, len(sel.Cols))
@ -152,8 +156,8 @@ func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.C
for _, id := range sel.Children {
child := &c.s[id]
rel, ok := c.schema.RelMap[child.RelID]
if !ok {
rel, err := c.schema.GetRel(child.Table, ti.Name)
if err != nil {
skipped |= (1 << uint(id))
continue
}
@ -183,12 +187,12 @@ func (c *compilerContext) processChildren(sel *qcode.Select) (uint32, []*qcode.C
return skipped, cols
}
func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) {
skipped, childCols := c.processChildren(sel)
func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint32, error) {
skipped, childCols := c.processChildren(sel, ti)
hasOrder := len(sel.OrderBy) != 0
// SELECT
if sel.AsList {
if ti.Singular == false {
//fmt.Fprintf(w, `SELECT coalesce(json_agg("%s"`, c.sel.Table)
c.w.WriteString(`SELECT coalesce(json_agg("`)
c.w.WriteString(sel.Table)
@ -246,7 +250,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) {
// END-SELECT
// FROM (SELECT .... )
err = c.renderBaseSelect(sel, childCols, skipped)
err = c.renderBaseSelect(sel, ti, childCols, skipped)
if err != nil {
return skipped, err
}
@ -255,7 +259,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select) (uint32, error) {
return skipped, nil
}
func (c *compilerContext) renderSelectClose(sel *qcode.Select) error {
func (c *compilerContext) renderSelectClose(sel *qcode.Select, ti *DBTableInfo) error {
hasOrder := len(sel.OrderBy) != 0
if hasOrder {
@ -270,6 +274,10 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select) error {
c.w.WriteString(` LIMIT ('`)
c.w.WriteString(sel.Paging.Limit)
c.w.WriteString(`') :: integer`)
} else if ti.Singular {
c.w.WriteString(` LIMIT ('1') :: integer`)
} else {
c.w.WriteString(` LIMIT ('20') :: integer`)
}
@ -281,7 +289,7 @@ func (c *compilerContext) renderSelectClose(sel *qcode.Select) error {
c.w.WriteString(`') :: integer`)
}
if sel.AsList {
if ti.Singular == false {
//fmt.Fprintf(w, `) AS "%s_%d"`, c.sel.Table, c.sel.ID)
c.w.WriteString(`)`)
aliasWithID(c.w, sel.Table, sel.ID)
@ -304,16 +312,21 @@ func (c *compilerContext) renderJoinClose(sel *qcode.Select) error {
}
func (c *compilerContext) renderJoinTable(sel *qcode.Select) {
rel, ok := c.schema.RelMap[sel.RelID]
if !ok {
panic(errors.New("no relationship found"))
parent := &c.s[sel.ParentID]
rel, err := c.schema.GetRel(sel.Table, parent.Table)
if err != nil {
panic(err)
}
if rel.Type != RelOneToManyThrough {
return
}
parent := &c.s[sel.ParentID]
pt, err := c.schema.GetTable(parent.Table)
if err != nil {
return
}
//fmt.Fprintf(w, ` LEFT OUTER JOIN "%s" ON (("%s"."%s") = ("%s_%d"."%s"))`,
//rel.Through, rel.Through, rel.ColT, c.parent.Table, c.parent.ID, rel.Col1)
@ -322,7 +335,7 @@ func (c *compilerContext) renderJoinTable(sel *qcode.Select) {
c.w.WriteString(`" ON ((`)
colWithTable(c.w, rel.Through, rel.ColT)
c.w.WriteString(`) = (`)
colWithTableID(c.w, parent.Table, parent.ID, rel.Col1)
colWithTableID(c.w, pt.Name, parent.ID, rel.Col1)
c.w.WriteString(`))`)
}
@ -343,8 +356,8 @@ func (c *compilerContext) renderRemoteRelColumns(sel *qcode.Select) {
for _, id := range sel.Children {
child := &c.s[id]
rel, ok := c.schema.RelMap[child.RelID]
if !ok || rel.Type != RelRemote {
rel, err := c.schema.GetRel(child.Table, sel.Table)
if err != nil || rel.Type != RelRemote {
continue
}
if i != 0 || len(sel.Cols) != 0 {
@ -380,15 +393,10 @@ func (c *compilerContext) renderJoinedColumns(sel *qcode.Select, skipped uint32)
return nil
}
func (c *compilerContext) renderBaseSelect(sel *qcode.Select,
func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
childCols []*qcode.Column, skipped uint32) error {
var groupBy []int
ti, err := c.getTable(sel)
if err != nil {
return err
}
isRoot := sel.ID == 0
isFil := sel.Where != nil
isSearch := sel.Args["search"] != nil
@ -473,19 +481,30 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select,
}
c.w.WriteString(` FROM `)
if tn, ok := c.tmap[sel.Table]; ok {
if c.schema.IsAlias(sel.Table) || ti.Singular {
//fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Table)
colWithAlias(c.w, tn, sel.Table)
tableWithAlias(c.w, ti.Name, sel.Table)
} else {
//fmt.Fprintf(w, ` FROM "%s"`, c.sel.Table)
c.w.WriteString(`"`)
c.w.WriteString(sel.Table)
c.w.WriteString(ti.Name)
c.w.WriteString(`"`)
}
// if tn, ok := c.tmap[sel.Table]; ok {
// //fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Table)
// tableWithAlias(c.w, ti.Name, sel.Table)
// } else {
// //fmt.Fprintf(w, ` FROM "%s"`, c.sel.Table)
// c.w.WriteString(`"`)
// c.w.WriteString(sel.Table)
// c.w.WriteString(`"`)
// }
if isRoot && isFil {
c.w.WriteString(` WHERE (`)
if err := c.renderWhere(sel); err != nil {
if err := c.renderWhere(sel, ti); err != nil {
return err
}
c.w.WriteString(`)`)
@ -499,7 +518,7 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select,
if isFil {
c.w.WriteString(` AND `)
if err := c.renderWhere(sel); err != nil {
if err := c.renderWhere(sel, ti); err != nil {
return err
}
}
@ -525,6 +544,10 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select,
c.w.WriteString(` LIMIT ('`)
c.w.WriteString(sel.Paging.Limit)
c.w.WriteString(`') :: integer`)
} else if ti.Singular {
c.w.WriteString(` LIMIT ('1') :: integer`)
} else {
c.w.WriteString(` LIMIT ('20') :: integer`)
}
@ -562,13 +585,13 @@ func (c *compilerContext) renderOrderByColumns(sel *qcode.Select) {
}
func (c *compilerContext) renderRelationship(sel *qcode.Select) {
rel, ok := c.schema.RelMap[sel.RelID]
if !ok {
panic(errors.New("no relationship found"))
}
parent := c.s[sel.ParentID]
rel, err := c.schema.GetRel(sel.Table, parent.Table)
if err != nil {
panic(err)
}
switch rel.Type {
case RelBelongTo:
//fmt.Fprintf(w, `(("%s"."%s") = ("%s_%d"."%s"))`,
@ -599,18 +622,13 @@ func (c *compilerContext) renderRelationship(sel *qcode.Select) {
}
}
func (c *compilerContext) renderWhere(sel *qcode.Select) error {
func (c *compilerContext) renderWhere(sel *qcode.Select, ti *DBTableInfo) error {
st := util.NewStack()
if sel.Where != nil {
st.Push(sel.Where)
}
ti, err := c.getTable(sel)
if err != nil {
return err
}
for {
if st.Len() == 0 {
break
@ -909,6 +927,14 @@ func colWithAlias(w *bytes.Buffer, col, alias string) {
w.WriteString(`"`)
}
func tableWithAlias(w *bytes.Buffer, table, alias string) {
w.WriteString(`"`)
w.WriteString(table)
w.WriteString(`" AS "`)
w.WriteString(alias)
w.WriteString(`"`)
}
func colWithTable(w *bytes.Buffer, table, col string) {
w.WriteString(`"`)
w.WriteString(table)
@ -987,3 +1013,11 @@ func int2string(w *bytes.Buffer, val int32) {
w.WriteByte(charset[d])
}
}
func relID(h *xxhash.Digest, child, parent string) uint64 {
h.WriteString(child)
h.WriteString(parent)
v := h.Sum64()
h.Reset()
return v
}

View File

@ -100,12 +100,17 @@ func TestMain(m *testing.M) {
}
schema := &DBSchema{
Tables: make(map[string]*DBTableInfo),
RelMap: make(map[uint64]*DBRel),
t: make(map[string]*DBTableInfo),
rm: make(map[string]map[string]*DBRel),
al: make(map[string]struct{}),
}
aliases := map[string][]string{
"users": []string{"mes"},
}
for i, t := range tables {
schema.updateSchema(t, columns[i])
schema.updateSchema(t, columns[i], aliases)
}
vars := NewVariables(map[string]string{
@ -115,9 +120,6 @@ func TestMain(m *testing.M) {
pcompile = NewCompiler(Config{
Schema: schema,
Vars: vars,
TableMap: map[string]string{
"mes": "users",
},
})
os.Exit(m.Run())
@ -260,7 +262,7 @@ func fetchByID(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('product', products) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name") AS "sel_0")) AS "products" FROM (SELECT "products"."id", "products"."name" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8)) AND (("id") = (15))) LIMIT ('1') :: integer) AS "products_0" LIMIT ('1') :: integer) AS "done_1337";`
sql := `SELECT json_object_agg('product', product) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "product_0"."id" AS "id", "product_0"."name" AS "name") AS "sel_0")) AS "product" FROM (SELECT "product"."id", "product"."name" FROM "products" AS "product" WHERE ((("product"."price") > (0)) AND (("product"."price") < (8)) AND (("id") = (15))) LIMIT ('1') :: integer) AS "product_0" LIMIT ('1') :: integer) AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {
@ -432,7 +434,7 @@ func queryWithVariables(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('product', products) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name") AS "sel_0")) AS "products" FROM (SELECT "products"."id", "products"."name" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8)) AND (("products"."price") = ('{{PRODUCT_PRICE}}')) AND (("id") = ('{{PRODUCT_ID}}'))) LIMIT ('1') :: integer) AS "products_0" LIMIT ('1') :: integer) AS "done_1337";`
sql := `SELECT json_object_agg('product', product) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "product_0"."id" AS "id", "product_0"."name" AS "name") AS "sel_0")) AS "product" FROM (SELECT "product"."id", "product"."name" FROM "products" AS "product" WHERE ((("product"."price") > (0)) AND (("product"."price") < (8)) AND (("product"."price") = ('{{PRODUCT_PRICE}}')) AND (("id") = ('{{PRODUCT_ID}}'))) LIMIT ('1') :: integer) AS "product_0" LIMIT ('1') :: integer) AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {
@ -451,7 +453,7 @@ func syntheticTables(t *testing.T) {
}
}`
sql := `SELECT json_object_agg('me', mes) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "mes_0"."email" AS "email") AS "sel_0")) AS "mes" FROM (SELECT "mes"."email" FROM "users" AS "mes" WHERE ((("mes"."id") = ('{{user_id}}'))) LIMIT ('1') :: integer) AS "mes_0" LIMIT ('1') :: integer) AS "done_1337";`
sql := `SELECT json_object_agg('me', me) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "me_0"."email" AS "email") AS "sel_0")) AS "me" FROM (SELECT "me"."email" FROM "users" AS "me" WHERE ((("me"."id") = ('{{user_id}}'))) LIMIT ('1') :: integer) AS "me_0" LIMIT ('1') :: integer) AS "done_1337";`
resSQL, err := compileGQLToPSQL(gql)
if err != nil {

View File

@ -4,166 +4,10 @@ import (
"fmt"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/go-pg/pg"
"github.com/gobuffalo/flect"
)
type DBSchema struct {
Tables map[string]*DBTableInfo
RelMap map[uint64]*DBRel
}
type DBTableInfo struct {
Name string
PrimaryCol string
TSVCol string
Columns map[string]*DBColumn
}
type RelType int
const (
RelBelongTo RelType = iota + 1
RelOneToMany
RelOneToManyThrough
RelRemote
)
type DBRel struct {
Type RelType
Through string
ColT string
Col1 string
Col2 string
}
func NewDBSchema(db *pg.DB) (*DBSchema, error) {
schema := &DBSchema{
Tables: make(map[string]*DBTableInfo),
RelMap: make(map[uint64]*DBRel),
}
tables, err := GetTables(db)
if err != nil {
return nil, err
}
for _, t := range tables {
cols, err := GetColumns(db, "public", t.Name)
if err != nil {
return nil, err
}
schema.updateSchema(t, cols)
}
return schema, nil
}
func (s *DBSchema) updateSchema(t *DBTable, cols []*DBColumn) {
// Current table
ti := &DBTableInfo{
Name: t.Name,
Columns: make(map[string]*DBColumn, len(cols)),
}
// Foreign key columns in current table
var jcols []*DBColumn
colByID := make(map[int]*DBColumn)
for i := range cols {
c := cols[i]
ti.Columns[strings.ToLower(c.Name)] = cols[i]
colByID[c.ID] = cols[i]
}
ct := strings.ToLower(t.Name)
s.Tables[ct] = ti
h := xxhash.New()
for _, c := range cols {
switch {
case c.Type == "tsvector":
s.Tables[ct].TSVCol = c.Name
case c.PrimaryKey:
s.Tables[ct].PrimaryCol = c.Name
case len(c.FKeyTable) != 0:
if len(c.FKeyColID) == 0 {
continue
}
// Foreign key column name
ft := strings.ToLower(c.FKeyTable)
fc, ok := colByID[c.FKeyColID[0]]
if !ok {
continue
}
// Belongs-to relation between current table and the
// table in the foreign key
rel1 := &DBRel{RelBelongTo, "", "", c.Name, fc.Name}
s.RelMap[relID(h, ct, ft)] = rel1
// One-to-many relation between the foreign key table and the
// the current table
rel2 := &DBRel{RelOneToMany, "", "", fc.Name, c.Name}
s.RelMap[relID(h, ft, ct)] = rel2
jcols = append(jcols, c)
}
}
// If table contains multiple foreign key columns it's a possible
// join table for many-to-many relationships or multiple one-to-many
// relations
// Below one-to-many relations use the current table as the
// join table aka through table.
if len(jcols) > 1 {
for i := range jcols {
for n := range jcols {
if n != i {
s.updateSchemaOTMT(h, ct, jcols[i], jcols[n], colByID)
}
}
}
}
}
func (s *DBSchema) updateSchemaOTMT(
h *xxhash.Digest,
ct string,
col1, col2 *DBColumn,
colByID map[int]*DBColumn) {
t1 := strings.ToLower(col1.FKeyTable)
t2 := strings.ToLower(col2.FKeyTable)
fc1, ok := colByID[col1.FKeyColID[0]]
if !ok {
return
}
fc2, ok := colByID[col2.FKeyColID[0]]
if !ok {
return
}
// One-to-many-through relation between 1nd foreign key table and the
// 2nd foreign key table
//rel1 := &DBRel{RelOneToManyThrough, ct, fc1.Name, col1.Name}
rel1 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name, col1.Name}
s.RelMap[relID(h, t1, t2)] = rel1
// One-to-many-through relation between 2nd foreign key table and the
// 1nd foreign key table
//rel2 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name}
rel2 := &DBRel{RelOneToManyThrough, ct, col1.Name, fc1.Name, col2.Name}
s.RelMap[relID(h, t2, t1)] = rel2
}
type DBTable struct {
Name string `sql:"name"`
Type string `sql:"type"`
@ -255,18 +99,231 @@ WHERE c.relkind = 'r'::char
return t, nil
}
type DBSchema struct {
t map[string]*DBTableInfo
rm map[string]map[string]*DBRel
al map[string]struct{}
}
type DBTableInfo struct {
Name string
Singular bool
PrimaryCol string
TSVCol string
Columns map[string]*DBColumn
}
type RelType int
const (
RelBelongTo RelType = iota + 1
RelOneToMany
RelOneToManyThrough
RelRemote
)
type DBRel struct {
Type RelType
Through string
ColT string
Col1 string
Col2 string
}
func NewDBSchema(db *pg.DB, aliases map[string][]string) (*DBSchema, error) {
schema := &DBSchema{
t: make(map[string]*DBTableInfo),
rm: make(map[string]map[string]*DBRel),
al: make(map[string]struct{}),
}
tables, err := GetTables(db)
if err != nil {
return nil, err
}
for _, t := range tables {
cols, err := GetColumns(db, "public", t.Name)
if err != nil {
return nil, err
}
schema.updateSchema(t, cols, aliases)
}
return schema, nil
}
func (s *DBSchema) updateSchema(
t *DBTable,
cols []*DBColumn,
aliases map[string][]string) {
// Foreign key columns in current table
colByID := make(map[int]*DBColumn)
columns := make(map[string]*DBColumn, len(cols))
for i := range cols {
c := cols[i]
columns[strings.ToLower(c.Name)] = cols[i]
colByID[c.ID] = cols[i]
}
singular := strings.ToLower(flect.Singularize(t.Name))
s.t[singular] = &DBTableInfo{
Name: t.Name,
Singular: true,
Columns: columns,
}
plural := strings.ToLower(flect.Pluralize(t.Name))
s.t[plural] = &DBTableInfo{
Name: t.Name,
Singular: false,
Columns: columns,
}
ct := strings.ToLower(t.Name)
if al, ok := aliases[ct]; ok {
for i := range al {
k1 := flect.Singularize(al[i])
s.t[k1] = s.t[singular]
k2 := flect.Pluralize(al[i])
s.t[k2] = s.t[plural]
s.al[k1] = struct{}{}
s.al[k2] = struct{}{}
}
}
jcols := make([]*DBColumn, 0, len(cols))
for _, c := range cols {
switch {
case c.Type == "tsvector":
s.t[singular].TSVCol = c.Name
s.t[plural].TSVCol = c.Name
case c.PrimaryKey:
s.t[singular].PrimaryCol = c.Name
s.t[plural].PrimaryCol = c.Name
case len(c.FKeyTable) != 0:
if len(c.FKeyColID) == 0 {
continue
}
// Foreign key column name
ft := strings.ToLower(c.FKeyTable)
fc, ok := colByID[c.FKeyColID[0]]
if !ok {
continue
}
// Belongs-to relation between current table and the
// table in the foreign key
rel1 := &DBRel{RelBelongTo, "", "", c.Name, fc.Name}
s.SetRel(ct, ft, rel1)
// One-to-many relation between the foreign key table and the
// the current table
rel2 := &DBRel{RelOneToMany, "", "", fc.Name, c.Name}
s.SetRel(ft, ct, rel2)
jcols = append(jcols, c)
}
}
// If table contains multiple foreign key columns it's a possible
// join table for many-to-many relationships or multiple one-to-many
// relations
// Below one-to-many relations use the current table as the
// join table aka through table.
if len(jcols) > 1 {
for i := range jcols {
for n := range jcols {
if n != i {
s.updateSchemaOTMT(ct, jcols[i], jcols[n], colByID)
}
}
}
}
}
func (s *DBSchema) updateSchemaOTMT(
ct string,
col1, col2 *DBColumn,
colByID map[int]*DBColumn) {
t1 := strings.ToLower(col1.FKeyTable)
t2 := strings.ToLower(col2.FKeyTable)
fc1, ok := colByID[col1.FKeyColID[0]]
if !ok {
return
}
fc2, ok := colByID[col2.FKeyColID[0]]
if !ok {
return
}
// One-to-many-through relation between 1nd foreign key table and the
// 2nd foreign key table
//rel1 := &DBRel{RelOneToManyThrough, ct, fc1.Name, col1.Name}
rel1 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name, col1.Name}
s.SetRel(t1, t2, rel1)
// One-to-many-through relation between 2nd foreign key table and the
// 1nd foreign key table
//rel2 := &DBRel{RelOneToManyThrough, ct, col2.Name, fc2.Name}
rel2 := &DBRel{RelOneToManyThrough, ct, col1.Name, fc1.Name, col2.Name}
s.SetRel(t2, t1, rel2)
}
func (s *DBSchema) GetTable(table string) (*DBTableInfo, error) {
t, ok := s.Tables[table]
t, ok := s.t[table]
if !ok {
return nil, fmt.Errorf("unknown table '%s'", table)
}
return t, nil
}
func relID(h *xxhash.Digest, child, parent string) uint64 {
h.WriteString(child)
h.WriteString(parent)
v := h.Sum64()
h.Reset()
return v
func (s *DBSchema) SetRel(child, parent string, rel *DBRel) error {
sc := strings.ToLower(flect.Singularize(child))
pc := strings.ToLower(flect.Pluralize(child))
if _, ok := s.rm[sc]; !ok {
s.rm[sc] = make(map[string]*DBRel)
}
if _, ok := s.rm[pc]; !ok {
s.rm[pc] = make(map[string]*DBRel)
}
sp := strings.ToLower(flect.Singularize(parent))
pp := strings.ToLower(flect.Pluralize(parent))
s.rm[sc][sp] = rel
s.rm[sc][pp] = rel
s.rm[pc][sp] = rel
s.rm[pc][pp] = rel
return nil
}
func (s *DBSchema) GetRel(child, parent string) (*DBRel, error) {
rel, ok := s.rm[child][parent]
if !ok {
return nil, fmt.Errorf("unknown relationship '%s' -> '%s'",
child, parent)
}
return rel, nil
}
func (s *DBSchema) IsAlias(name string) bool {
_, ok := s.al[name]
return ok
}

View File

@ -303,6 +303,8 @@ func lexName(l *lexer) stateFn {
l.backup()
v := l.current()
lowercase(l.input, s, e)
if len(v) == 0 {
switch {
case strings.EqualFold(v, "query"):

View File

@ -5,7 +5,6 @@ import (
"fmt"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/dosco/super-graph/util"
"github.com/gobuffalo/flect"
)
@ -31,11 +30,8 @@ type Column struct {
type Select struct {
ID int32
ParentID int32
RelID uint64
Args map[string]*Node
AsList bool
Table string
Singular string
FieldName string
Cols []Column
Where *Exp
@ -163,7 +159,12 @@ func NewCompiler(c Config) (*Compiler, error) {
if err != nil {
return nil, err
}
fm[strings.ToLower(k)] = fil
k1 := strings.ToLower(k)
singular := flect.Singularize(k1)
plural := flect.Pluralize(k1)
fm[singular] = fil
fm[plural] = fil
}
return &Compiler{fl, fm, bl, c.KeepArgs}, nil
@ -202,7 +203,6 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
selects := make([]Select, 0, 5)
st := util.NewStack()
h := xxhash.New()
if len(op.Fields) == 0 {
return nil, errors.New("empty query")
@ -226,46 +226,31 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
}
field := &op.Fields[fid]
fn := strings.ToLower(field.Name)
if _, ok := com.bl[fn]; ok {
tn := strings.ToLower(field.Name)
if _, ok := com.bl[tn]; ok {
continue
}
tn := flect.Pluralize(fn)
s := Select{
selects = append(selects, Select{
ID: id,
ParentID: parentID,
Table: tn,
Children: make([]int32, 0, 5),
}
})
s := &selects[(len(selects) - 1)]
if s.ID != 0 {
p := &selects[s.ParentID]
p.Children = append(p.Children, s.ID)
s.RelID = relID(h, tn, p.Table)
}
if fn == tn {
s.Singular = flect.Singularize(fn)
} else {
s.Singular = fn
}
if fn == s.Table {
s.AsList = true
} else {
s.Paging.Limit = "1"
}
if len(field.Alias) != 0 {
s.FieldName = field.Alias
} else if s.AsList {
s.FieldName = s.Table
} else {
s.FieldName = s.Singular
s.FieldName = s.Table
}
err := com.compileArgs(&s, field.Args)
err := com.compileArgs(s, field.Args)
if err != nil {
return nil, err
}
@ -296,7 +281,6 @@ func (com *Compiler) compileQuery(op *Operation) (*Query, error) {
s.Cols = append(s.Cols, col)
}
selects = append(selects, s)
id++
}
@ -858,14 +842,6 @@ func buildPath(a []string) string {
return b.String()
}
func relID(h *xxhash.Digest, child, parent string) uint64 {
h.WriteString(child)
h.WriteString(parent)
v := h.Sum64()
h.Reset()
return v
}
func (t ExpOp) String() string {
var v string

View File

@ -13,7 +13,7 @@ services:
ports:
- "8080:8080"
web:
rails_app:
image: dosco/super-graph-demo:latest
environment:
RAILS_ENV: "development"

View File

@ -1,6 +1,8 @@
package serv
import (
"strings"
"github.com/gobuffalo/flect"
)
@ -84,8 +86,8 @@ type configRemote struct {
} `mapstructure:"set_headers"`
}
func (c *config) getAliasMap() map[string]string {
m := make(map[string]string, len(c.DB.Tables))
func (c *config) getAliasMap() map[string][]string {
m := make(map[string][]string, len(c.DB.Tables))
for i := range c.DB.Tables {
t := c.DB.Tables[i]
@ -93,7 +95,9 @@ func (c *config) getAliasMap() map[string]string {
if len(t.Table) == 0 {
continue
}
m[flect.Pluralize(t.Name)] = t.Table
k := strings.ToLower(t.Table)
m[k] = append(m[k], strings.ToLower(t.Name))
}
return m
}
@ -107,12 +111,16 @@ func (c *config) getFilterMap() map[string][]string {
if len(t.Filter) == 0 {
continue
}
name := flect.Pluralize(t.Name)
singular := flect.Singularize(t.Name)
plural := flect.Pluralize(t.Name)
if t.Filter[0] == "none" {
m[name] = []string{}
m[singular] = []string{}
m[plural] = []string{}
} else {
m[name] = t.Filter
m[singular] = t.Filter
m[plural] = t.Filter
}
}

View File

@ -231,7 +231,6 @@ func (c *coreContext) resolveRemotes(
to[n] = jsn.Field{[]byte(s.FieldName), ob.Bytes()}
}(i, id, s)
fmt.Println(">>>", i)
}
wg.Wait()

View File

@ -22,16 +22,21 @@ type resolvFn struct {
Fn func(r *http.Request, id []byte) ([]byte, error)
}
func initResolvers() {
func initResolvers() error {
rmap = make(map[uint64]*resolvFn)
for _, t := range conf.DB.Tables {
initRemotes(t)
err := initRemotes(t)
if err != nil {
return err
}
}
return nil
}
func initRemotes(t configTable) {
func initRemotes(t configTable) error {
h := xxhash.New()
var err error
for _, r := range t.Remotes {
// defines the table column to be used as an id in the
@ -41,24 +46,26 @@ func initRemotes(t configTable) {
// if no table column specified in the config then
// use the primary key of the table as the id
if len(idcol) == 0 {
idcol = pcompile.IDColumn(t.Name)
idcol, err = pcompile.IDColumn(t.Name)
if err != nil {
return err
}
}
idk := fmt.Sprintf("__%s_%s", t.Name, idcol)
// register a relationship between the remote data
// and the database table
h.WriteString(strings.ToLower(r.Name))
h.WriteString(t.Name)
key := h.Sum64()
h.Reset()
val := &psql.DBRel{
Type: psql.RelRemote,
Col1: idcol,
Col2: idk,
}
pcompile.AddRelationship(key, val)
err := pcompile.AddRelationship(strings.ToLower(r.Name), t.Name, val)
if err != nil {
return err
}
// the function thats called to resolve this remote
// data request
@ -81,6 +88,8 @@ func initRemotes(t configTable) {
// index resolver obj by IDField
rmap[xxhash.Sum64(rf.IDField)] = rf
}
return nil
}
func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
@ -88,7 +97,8 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
client := &http.Client{}
fn := func(inReq *http.Request, id []byte) ([]byte, error) {
req, err := http.NewRequest("GET", fmt.Sprintf(reqURL, id), nil)
uri := fmt.Sprintf(reqURL, id)
req, err := http.NewRequest("GET", uri, nil)
if err != nil {
return nil, err
}
@ -107,6 +117,7 @@ func buildFn(r configRemote) func(*http.Request, []byte) ([]byte, error) {
res, err := client.Do(req)
if err != nil {
logger.Error().Err(err).Msgf("Failed to connect to: %s", uri)
return nil, err
}
defer res.Body.Close()

View File

@ -150,7 +150,7 @@ func initDB(c *config) (*pg.DB, error) {
}
func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
schema, err := psql.NewDBSchema(db)
schema, err := psql.NewDBSchema(db, c.getAliasMap())
if err != nil {
return nil, nil, err
}
@ -169,7 +169,6 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
pc := psql.NewCompiler(psql.Config{
Schema: schema,
Vars: c.DB.Variables,
TableMap: c.getAliasMap(),
})
return qc, pc, nil
@ -182,20 +181,22 @@ func Init() {
conf, err = initConf()
if err != nil {
logger.Fatal().Err(err)
logger.Fatal().Err(err).Msg("failed to read config")
}
db, err = initDB(conf)
if err != nil {
logger.Fatal().Err(err)
logger.Fatal().Err(err).Msg("failed to connect to database")
}
qcompile, pcompile, err = initCompilers(conf)
if err != nil {
logger.Fatal().Err(err)
logger.Fatal().Err(err).Msg("failed to connect to database")
}
initResolvers()
if err := initResolvers(); err != nil {
logger.Fatal().Err(err).Msg("failed to initialized resolvers")
}
startHTTP()
}
@ -216,21 +217,21 @@ func startHTTP() {
<-sigint
if err := srv.Shutdown(context.Background()); err != nil {
logger.Printf("http: %v", err)
logger.Error().Err(err).Msg("shutdown signal received")
}
close(idleConnsClosed)
}()
srv.RegisterOnShutdown(func() {
if err := db.Close(); err != nil {
logger.Error().Err(err)
logger.Error().Err(err).Msg("db closed")
}
})
fmt.Printf("%s listening on %s (%s)\n", serverName, conf.HostPort, conf.Env)
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
fmt.Println(err)
logger.Error().Err(err).Msg("server closed")
}
<-idleConnsClosed