diff --git a/docs/guide.md b/docs/guide.md index 085ea8b..f32ff9a 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -906,7 +906,7 @@ mutation { } ``` -### Nested Updates +### Nested Update Update a product item first and then assign it to a user @@ -966,7 +966,7 @@ mutation { } ``` -## Using variables +## Using Variables Variables (`$product_id`) and their values (`"product_id": 5`) can be passed along side the GraphQL query. Using variables makes for better client side code as well as improved server side SQL query caching. The build-in web-ui also supports setting variables. Not having to manipulate your GraphQL query string to insert values into it makes for cleaner and better client side code. @@ -988,6 +988,70 @@ fetch('http://localhost:8080/api/v1/graphql', { .then(res => console.log(res.data)); ``` +## Advanced Columns + +The ablity to have `JSON/JSONB` and `Array` columns is often considered in the top most useful features of Postgres. There are many cases where using an array or a json column saves space and reduces complexity in your app. The only issue with these columns is the really that your SQL queries can get harder to write and maintain. + +Super Graph steps in here to help you by supporting these columns right out of the box. It allows you to work with these columns just like you would with tables. Joining data against or modifying array columns using the `connect` or `disconnect` keywords in mutations is fully supported. Another very useful feature is the ability to treat `json` or `binary json (jsonb)` columns as seperate tables, even using them in nested queries joining against related tables. To replicate these features on your own will take a lot of complex SQL. Using Super Graph means you don't have to deal with any of this it just works. + +### Array Columns + +Configure a relationship between an array column `tag_ids` which contains integer id's for tags and the column `id` in the table `tags`. + +```yaml +tables: + - name: posts + columns: + - name: tag_ids + related_to: tags.id + +``` + +```graphql +query { + posts { + title + tags { + name + image + } + } +} +``` + +### JSON Column + +Configure a JSON column called `tag_count` in the table `products` into a seperate table. This JSON column contains a json array of objects each with a tag id and a count of the number of times the tag was used. As a seperate table you can nest it into your GraphQL query and treat it like table using any of the standard features like `order_by`, `limit`, `where clauses`, etc. + +The configuration below tells Super Graph to create a synthetic table called `tag_count` using the column `tag_count` from the `products` table. And that this new table has two columns `tag_id` and `count` of the listed types and with the defined relationships. + +```yaml +tables: + - name: tag_count + table: products + columns: + - name: tag_id + type: bigint + related_to: tags.id + - name: count + type: int +``` + +```graphql +query { + products { + name + tag_counts { + count + tag { + name + } + } + } +} +``` + + ## Full text search Every app these days needs search. Enought his often means reaching for something heavy like Solr. While this will work why add complexity to your infrastructure when Postgres has really great diff --git a/psql/psql_test.go b/psql/psql_test.go index ee5001e..2146d46 100644 --- a/psql/psql_test.go +++ b/psql/psql_test.go @@ -134,6 +134,7 @@ func TestMain(m *testing.M) { DBTable{Name: "products", Type: "table"}, DBTable{Name: "purchases", Type: "table"}, DBTable{Name: "tags", Type: "table"}, + DBTable{Name: "tag_count", Type: "json"}, } columns := [][]DBColumn{ @@ -169,7 +170,8 @@ func TestMain(m *testing.M) { DBColumn{ID: 6, Name: "created_at", Type: "timestamp without time zone", NotNull: true, PrimaryKey: false, UniqueKey: false}, DBColumn{ID: 7, Name: "updated_at", Type: "timestamp without time zone", NotNull: true, PrimaryKey: false, UniqueKey: false}, DBColumn{ID: 8, Name: "tsv", Type: "tsvector", NotNull: false, PrimaryKey: false, UniqueKey: false}, - DBColumn{ID: 9, Name: "tags", Type: "text[]", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{3}, Array: true}}, + DBColumn{ID: 9, Name: "tags", Type: "text[]", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{3}, Array: true}, + DBColumn{ID: 9, Name: "tag_count", Type: "json", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tag_count", FKeyColID: []int16{}}}, []DBColumn{ DBColumn{ID: 1, Name: "id", Type: "bigint", NotNull: true, PrimaryKey: true, UniqueKey: true}, DBColumn{ID: 2, Name: "customer_id", Type: "bigint", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "customers", FKeyColID: []int16{1}}, @@ -182,6 +184,9 @@ func TestMain(m *testing.M) { DBColumn{ID: 1, Name: "id", Type: "bigint", NotNull: true, PrimaryKey: true, UniqueKey: true}, DBColumn{ID: 2, Name: "name", Type: "text", NotNull: false, PrimaryKey: false, UniqueKey: false}, DBColumn{ID: 3, Name: "slug", Type: "text", NotNull: false, PrimaryKey: false, UniqueKey: false}}, + []DBColumn{ + DBColumn{ID: 1, Name: "tag_id", Type: "bigint", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{1}}, + DBColumn{ID: 2, Name: "count", Type: "int", NotNull: false, PrimaryKey: false, UniqueKey: false}}, } for i := range tables { diff --git a/psql/query.go b/psql/query.go index 1cdc881..c76f02e 100644 --- a/psql/query.go +++ b/psql/query.go @@ -224,7 +224,7 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w io.Writer) (uint32, error) { return ignored, nil } -func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column) { +func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column, error) { var skipped uint32 cols := make([]*qcode.Column, 0, len(sel.Cols)) @@ -243,40 +243,63 @@ func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (u rel, err := c.schema.GetRel(child.Name, ti.Name) if err != nil { - skipped |= (1 << uint(id)) - continue + return 0, nil, err + //skipped |= (1 << uint(id)) + //continue } switch rel.Type { case RelOneToOne, RelOneToMany: if _, ok := colmap[rel.Right.Col]; !ok { cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Right.Col, FieldName: rel.Right.Col}) + colmap[rel.Right.Col] = struct{}{} } - colmap[rel.Right.Col] = struct{}{} case RelOneToManyThrough: if _, ok := colmap[rel.Left.Col]; !ok { cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Left.Col}) + colmap[rel.Left.Col] = struct{}{} + } + + case RelEmbedded: + if _, ok := colmap[rel.Left.Col]; !ok { + cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Left.Col}) + colmap[rel.Left.Col] = struct{}{} } - colmap[rel.Left.Col] = struct{}{} case RelRemote: if _, ok := colmap[rel.Left.Col]; !ok { cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Right.Col}) + colmap[rel.Left.Col] = struct{}{} + skipped |= (1 << uint(id)) } - colmap[rel.Left.Col] = struct{}{} - skipped |= (1 << uint(id)) default: - skipped |= (1 << uint(id)) + return 0, nil, fmt.Errorf("unknown relationship %s", rel) + //skipped |= (1 << uint(id)) } } - return skipped, cols + return skipped, cols, nil } func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint32, error) { - skipped, childCols := c.processChildren(sel, ti) + var rel *DBRel + var err error + + if sel.ParentID != -1 { + parent := c.s[sel.ParentID] + + rel, err = c.schema.GetRel(ti.Name, parent.Name) + if err != nil { + return 0, err + } + } + + skipped, childCols, err := c.processChildren(sel, ti) + if err != nil { + return 0, err + } hasOrder := len(sel.OrderBy) != 0 // SELECT @@ -288,9 +311,8 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint io.WriteString(c.w, `"`) if hasOrder { - err := c.renderOrderBy(sel, ti) - if err != nil { - return skipped, err + if err := c.renderOrderBy(sel, ti); err != nil { + return 0, err } } @@ -319,8 +341,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint c.renderRemoteRelColumns(sel, ti) - err := c.renderJoinedColumns(sel, ti, skipped) - if err != nil { + if err = c.renderJoinedColumns(sel, ti, skipped); err != nil { return skipped, err } @@ -339,7 +360,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint // END-SELECT // FROM (SELECT .... ) - err = c.renderBaseSelect(sel, ti, childCols, skipped) + err = c.renderBaseSelect(sel, ti, rel, childCols, skipped) if err != nil { return skipped, err } @@ -527,11 +548,11 @@ func (c *compilerContext) renderJoinedColumns(sel *qcode.Select, ti *DBTableInfo return nil } -func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, +func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, rel *DBRel, childCols []*qcode.Column, skipped uint32) error { var groupBy []int - isRoot := sel.ParentID == -1 + isRoot := (rel == nil) isFil := (sel.Where != nil && sel.Where.Op != qcode.OpNop) isSearch := sel.Args["search"] != nil isAgg := false @@ -682,10 +703,7 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, io.WriteString(c.w, ` FROM `) - //fmt.Fprintf(w, ` FROM "%s"`, c.sel.Name) - io.WriteString(c.w, `"`) - io.WriteString(c.w, ti.Name) - io.WriteString(c.w, `"`) + c.renderFrom(sel, ti, rel) // if tn, ok := c.tmap[sel.Name]; ok { // //fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Name) @@ -711,11 +729,9 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, } io.WriteString(c.w, ` WHERE (`) - if err := c.renderRelationship(sel, ti); err != nil { return err } - if isFil { io.WriteString(c.w, ` AND `) if err := c.renderWhere(sel, ti); err != nil { @@ -770,6 +786,44 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, return nil } +func (c *compilerContext) renderFrom(sel *qcode.Select, ti *DBTableInfo, rel *DBRel) error { + if rel != nil && rel.Type == RelEmbedded { + // json_to_recordset('[{"a":1,"b":[1,2,3],"c":"bar"}, {"a":2,"b":[1,2,3],"c":"bar"}]') as x(a int, b text, d text); + + io.WriteString(c.w, `"`) + io.WriteString(c.w, rel.Left.Table) + io.WriteString(c.w, `", `) + + io.WriteString(c.w, ti.Type) + io.WriteString(c.w, `_to_recordset(`) + colWithTable(c.w, rel.Left.Table, rel.Right.Col) + io.WriteString(c.w, `) AS `) + + io.WriteString(c.w, `"`) + io.WriteString(c.w, ti.Name) + io.WriteString(c.w, `"`) + + io.WriteString(c.w, `(`) + for i, col := range ti.Columns { + if i != 0 { + io.WriteString(c.w, `, `) + } + io.WriteString(c.w, col.Name) + io.WriteString(c.w, ` `) + io.WriteString(c.w, col.Type) + } + io.WriteString(c.w, `)`) + + } else { + //fmt.Fprintf(w, ` FROM "%s"`, c.sel.Name) + io.WriteString(c.w, `"`) + io.WriteString(c.w, ti.Name) + io.WriteString(c.w, `"`) + } + + return nil +} + func (c *compilerContext) renderOrderByColumns(sel *qcode.Select, ti *DBTableInfo) { colsRendered := len(sel.Cols) != 0 @@ -852,7 +906,13 @@ func (c *compilerContext) renderRelationshipByName(table, parent string, id int3 io.WriteString(c.w, `) = (`) colWithTable(c.w, rel.Through, rel.Right.Col) } + + case RelEmbedded: + colWithTable(c.w, rel.Left.Table, rel.Left.Col) + io.WriteString(c.w, `) = (`) + colWithTableID(c.w, parent, id, rel.Left.Col) } + io.WriteString(c.w, `))`) return nil diff --git a/psql/query_test.go b/psql/query_test.go index 12ee395..b441a95 100644 --- a/psql/query_test.go +++ b/psql/query_test.go @@ -463,6 +463,32 @@ func multiRoot(t *testing.T) { } } +func jsonColumnAsTable(t *testing.T) { + gql := `query { + products { + id + name + tag_count { + count + tags { + name + } + } + } + }` + + sql := `SELECT json_object_agg('products', json_0) FROM (SELECT coalesce(json_agg("json_0"), '[]') AS "json_0" FROM (SELECT row_to_json((SELECT "json_row_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name", "tag_count_1_join"."json_1" AS "tag_count") AS "json_row_0")) AS "json_0" FROM (SELECT "products"."id", "products"."name" FROM "products" LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT row_to_json((SELECT "json_row_1" FROM (SELECT "tag_count_1"."count" AS "count", "tags_2_join"."json_2" AS "tags") AS "json_row_1")) AS "json_1" FROM (SELECT "tag_count"."count", "tag_count"."tag_id" FROM "products", json_to_recordset("products"."tag_count") AS "tag_count"(tag_id bigint, count int) WHERE ((("products"."id") = ("products_0"."id"))) LIMIT ('1') :: integer) AS "tag_count_1" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("json_2"), '[]') AS "json_2" FROM (SELECT row_to_json((SELECT "json_row_2" FROM (SELECT "tags_2"."name" AS "name") AS "json_row_2")) AS "json_2" FROM (SELECT "tags"."name" FROM "tags" WHERE ((("tags"."id") = ("tag_count_1"."tag_id"))) LIMIT ('20') :: integer) AS "tags_2" LIMIT ('20') :: integer) AS "json_agg_2") AS "tags_2_join" ON ('true') LIMIT ('1') :: integer) AS "tag_count_1_join" ON ('true') LIMIT ('20') :: integer) AS "json_agg_0") AS "sel_0"` + + resSQL, err := compileGQLToPSQL(gql, nil, "admin") + if err != nil { + t.Fatal(err) + } + + if string(resSQL) != sql { + t.Fatal(errNotExpected) + } +} + func skipUserIDForAnonRole(t *testing.T) { gql := `query { products { @@ -548,6 +574,7 @@ func TestCompileQuery(t *testing.T) { t.Run("queryWithVariables", queryWithVariables) t.Run("withWhereOnRelations", withWhereOnRelations) t.Run("multiRoot", multiRoot) + t.Run("jsonColumnAsTable", jsonColumnAsTable) t.Run("skipUserIDForAnonRole", skipUserIDForAnonRole) t.Run("blockedQuery", blockedQuery) t.Run("blockedFunctions", blockedFunctions) diff --git a/psql/schema.go b/psql/schema.go index 3005d9c..389a134 100644 --- a/psql/schema.go +++ b/psql/schema.go @@ -15,6 +15,7 @@ type DBSchema struct { type DBTableInfo struct { Name string + Type string Singular bool Columns []DBColumn PrimaryCol *DBColumn @@ -29,6 +30,7 @@ const ( RelOneToOne RelType = iota + 1 RelOneToMany RelOneToManyThrough + RelEmbedded RelRemote ) @@ -51,7 +53,6 @@ type DBRel struct { } func NewDBSchema(info *DBInfo, aliases map[string][]string) (*DBSchema, error) { - schema := &DBSchema{ t: make(map[string]*DBTableInfo), rm: make(map[string]map[string]*DBRel), @@ -83,6 +84,7 @@ func (s *DBSchema) addTable( singular := flect.Singularize(t.Key) s.t[singular] = &DBTableInfo{ Name: t.Name, + Type: t.Type, Singular: true, Columns: cols, ColMap: colmap, @@ -92,6 +94,7 @@ func (s *DBSchema) addTable( plural := flect.Pluralize(t.Key) s.t[plural] = &DBTableInfo{ Name: t.Name, + Type: t.Type, Singular: false, Columns: cols, ColMap: colmap, @@ -139,19 +142,43 @@ func (s *DBSchema) updateRelationships(t DBTable, cols []DBColumn) error { for i := range cols { c := cols[i] - if len(c.FKeyTable) == 0 || len(c.FKeyColID) == 0 { + if len(c.FKeyTable) == 0 { continue } // Foreign key column name ft := strings.ToLower(c.FKeyTable) - fcid := c.FKeyColID[0] ti, ok := s.t[ft] if !ok { return fmt.Errorf("invalid foreign key table '%s'", ft) } + // This is an embedded relationship like when a json/jsonb column + // is exposed as a table + if c.Name == c.FKeyTable && len(c.FKeyColID) == 0 { + rel := &DBRel{Type: RelEmbedded} + rel.Left.col = cti.PrimaryCol + rel.Left.Table = cti.Name + rel.Left.Col = cti.PrimaryCol.Name + + rel.Right.col = &c + rel.Right.Table = ti.Name + rel.Right.Col = c.Name + + if err := s.SetRel(ft, ct, rel); err != nil { + return err + } + continue + } + + if len(c.FKeyColID) == 0 { + continue + } + + // Foreign key column id + fcid := c.FKeyColID[0] + fc, ok := ti.ColIDMap[fcid] if !ok { return fmt.Errorf("invalid foreign key column id '%d' for table '%s'", diff --git a/psql/strings.go b/psql/strings.go index 451a739..b9c4292 100644 --- a/psql/strings.go +++ b/psql/strings.go @@ -12,6 +12,8 @@ func (rt RelType) String() string { return "one to many through" case RelRemote: return "remote" + case RelEmbedded: + return "embedded" } return "" } diff --git a/psql/tables.go b/psql/tables.go index 610198c..b9b88e9 100644 --- a/psql/tables.go +++ b/psql/tables.go @@ -62,6 +62,20 @@ func GetDBInfo(db *pgxpool.Pool) (*DBInfo, error) { return di, nil } +func (di *DBInfo) AddTable(t DBTable, cols []DBColumn) { + t.ID = di.Tables[len(di.Tables)-1].ID + + di.Tables = append(di.Tables, t) + di.colmap[t.Key] = make(map[string]*DBColumn, len(cols)) + + for i := range cols { + cols[i].ID = int16(i) + c := &cols[i] + di.colmap[t.Key][c.Key] = c + } + di.Columns = append(di.Columns, cols) +} + func (di *DBInfo) GetColumn(table, column string) (*DBColumn, bool) { v, ok := di.colmap[strings.ToLower(table)][strings.ToLower(column)] return v, ok diff --git a/serv/config.go b/serv/config.go index dba1ac2..7df9086 100644 --- a/serv/config.go +++ b/serv/config.go @@ -87,6 +87,7 @@ type config struct { type configColumn struct { Name string + Type string ForeignKey string `mapstructure:"related_to"` } @@ -313,7 +314,7 @@ func (c *config) getAliasMap() map[string][]string { for i := range c.Tables { t := c.Tables[i] - if len(t.Table) == 0 { + if len(t.Table) == 0 || len(t.Columns) != 0 { continue } diff --git a/serv/config_compile.go b/serv/config_compile.go index 36ad073..5bb6075 100644 --- a/serv/config_compile.go +++ b/serv/config_compile.go @@ -8,9 +8,61 @@ import ( "github.com/dosco/super-graph/qcode" ) +func addTables(c *config, di *psql.DBInfo) error { + for _, t := range c.Tables { + if len(t.Table) == 0 || len(t.Columns) == 0 { + continue + } + if err := addTable(di, t.Columns, t); err != nil { + return err + } + } + return nil +} + +func addTable(di *psql.DBInfo, cols []configColumn, t configTable) error { + bc, ok := di.GetColumn(t.Table, t.Name) + if !ok { + return fmt.Errorf( + "Column '%s' not found on table '%s'", + t.Name, t.Table) + } + + if bc.Type != "json" && bc.Type != "jsonb" { + return fmt.Errorf( + "Column '%s' in table '%s' is of type '%s'. Only JSON or JSONB is valid", + t.Name, t.Table, bc.Type) + } + + table := psql.DBTable{ + Name: t.Name, + Key: strings.ToLower(t.Name), + Type: bc.Type, + } + + columns := make([]psql.DBColumn, 0, len(cols)) + + for i := range cols { + c := cols[i] + columns = append(columns, psql.DBColumn{ + Name: c.Name, + Key: strings.ToLower(c.Name), + Type: c.Type, + }) + } + + di.AddTable(table, columns) + bc.FKeyTable = t.Name + + return nil +} + func addForeignKeys(c *config, di *psql.DBInfo) error { for _, t := range c.Tables { for _, c := range t.Columns { + if len(c.ForeignKey) == 0 { + continue + } if err := addForeignKey(di, c, t); err != nil { return err } @@ -23,7 +75,7 @@ func addForeignKey(di *psql.DBInfo, c configColumn, t configTable) error { c1, ok := di.GetColumn(t.Name, c.Name) if !ok { return fmt.Errorf( - "Invalid table '%s' or column '%s in config", + "Invalid table '%s' or column '%s' in config", t.Name, c.Name) } diff --git a/serv/serv.go b/serv/serv.go index 7873225..d5b94ef 100644 --- a/serv/serv.go +++ b/serv/serv.go @@ -21,6 +21,10 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) { return nil, nil, err } + if err = addTables(c, di); err != nil { + return nil, nil, err + } + if err = addForeignKeys(c, di); err != nil { return nil, nil, err }