Add ability to treat JSON/JSONB columns as tables

docs/guide.md

@@ -906,7 +906,7 @@ mutation {
 }
 ```
 
-### Nested Updates 
+### Nested Update 
 
 Update a product item first and then assign it to a user
 
@@ -966,7 +966,7 @@ mutation {
 }
 ```
 
-## Using variables
+## Using Variables
 
 Variables (`$product_id`) and their values (`"product_id": 5`) can be passed along side the GraphQL query. Using variables makes for better client side code as well as improved server side SQL query caching. The build-in web-ui also supports setting variables. Not having to manipulate your GraphQL query string to insert values into it makes for cleaner
 and better client side code.
@@ -988,6 +988,70 @@ fetch('http://localhost:8080/api/v1/graphql', {
 .then(res => console.log(res.data));
 ```
 
+## Advanced Columns
+
+The ablity to have `JSON/JSONB` and `Array` columns is often considered in the top most useful features of Postgres. There are many cases where using an array or a json column saves space and reduces complexity in your app. The only issue with these columns is the really that your SQL queries can get harder to write and maintain. 
+
+Super Graph steps in here to help you by supporting these columns right out of the box. It allows you to work with these columns just like you would with tables. Joining data against or modifying array columns using the `connect` or `disconnect` keywords in mutations is fully supported. Another very useful feature is the ability to treat `json` or `binary json (jsonb)` columns as seperate tables, even using them in nested queries joining against related tables. To replicate these features on your own will take a lot of complex SQL. Using Super Graph means you don't have to deal with any of this it just works.
+
+### Array Columns
+
+Configure a relationship between an array column `tag_ids` which contains integer id's for tags and the column `id` in the table `tags`.
+
+```yaml
+tables:
+  - name: posts
+    columns:
+      - name: tag_ids
+        related_to: tags.id
+
+```
+
+```graphql
+query {
+  posts {
+    title
+    tags {
+      name
+      image
+    }
+  }
+}
+```
+
+### JSON Column
+
+Configure a JSON column called `tag_count` in the table `products` into a seperate table. This JSON column contains a json array of objects each with a tag id and a count of the number of times the tag was used. As a seperate table you can nest it into your GraphQL query and treat it like table using any of the standard features like `order_by`, `limit`, `where clauses`, etc.
+
+The configuration below tells Super Graph to create a synthetic table called `tag_count` using the column `tag_count` from the `products` table. And that this new table has two columns `tag_id` and `count` of the listed types and with the defined relationships.
+
+```yaml
+tables:
+  - name: tag_count
+    table: products
+    columns:
+      - name: tag_id
+        type: bigint
+        related_to: tags.id
+      - name: count
+        type: int
+```
+
+```graphql
+query {
+  products {
+    name
+    tag_counts {
+      count
+      tag {
+        name
+      }
+    }
+  }
+}
+```
+
+
 ## Full text search
 
 Every app these days needs search. Enought his often means reaching for something heavy like Solr. While this will work why add complexity to your infrastructure when Postgres has really great

psql/psql_test.go

@@ -134,6 +134,7 @@ func TestMain(m *testing.M) {
 		DBTable{Name: "products", Type: "table"},
 		DBTable{Name: "purchases", Type: "table"},
 		DBTable{Name: "tags", Type: "table"},
+		DBTable{Name: "tag_count", Type: "json"},
 	}
 
 	columns := [][]DBColumn{
@@ -169,7 +170,8 @@ func TestMain(m *testing.M) {
 			DBColumn{ID: 6, Name: "created_at", Type: "timestamp without time zone", NotNull: true, PrimaryKey: false, UniqueKey: false},
 			DBColumn{ID: 7, Name: "updated_at", Type: "timestamp without time zone", NotNull: true, PrimaryKey: false, UniqueKey: false},
 			DBColumn{ID: 8, Name: "tsv", Type: "tsvector", NotNull: false, PrimaryKey: false, UniqueKey: false},
-			DBColumn{ID: 9, Name: "tags", Type: "text[]", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{3}, Array: true}},
+			DBColumn{ID: 9, Name: "tags", Type: "text[]", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{3}, Array: true},
+			DBColumn{ID: 9, Name: "tag_count", Type: "json", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tag_count", FKeyColID: []int16{}}},
 		[]DBColumn{
 			DBColumn{ID: 1, Name: "id", Type: "bigint", NotNull: true, PrimaryKey: true, UniqueKey: true},
 			DBColumn{ID: 2, Name: "customer_id", Type: "bigint", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "customers", FKeyColID: []int16{1}},
@@ -182,6 +184,9 @@ func TestMain(m *testing.M) {
 			DBColumn{ID: 1, Name: "id", Type: "bigint", NotNull: true, PrimaryKey: true, UniqueKey: true},
 			DBColumn{ID: 2, Name: "name", Type: "text", NotNull: false, PrimaryKey: false, UniqueKey: false},
 			DBColumn{ID: 3, Name: "slug", Type: "text", NotNull: false, PrimaryKey: false, UniqueKey: false}},
+		[]DBColumn{
+			DBColumn{ID: 1, Name: "tag_id", Type: "bigint", NotNull: false, PrimaryKey: false, UniqueKey: false, FKeyTable: "tags", FKeyColID: []int16{1}},
+			DBColumn{ID: 2, Name: "count", Type: "int", NotNull: false, PrimaryKey: false, UniqueKey: false}},
 	}
 
 	for i := range tables {

psql/query.go

@@ -224,7 +224,7 @@ func (co *Compiler) compileQuery(qc *qcode.QCode, w io.Writer) (uint32, error) {
 	return ignored, nil
 }
 
-func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column) {
+func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (uint32, []*qcode.Column, error) {
 	var skipped uint32
 
 	cols := make([]*qcode.Column, 0, len(sel.Cols))
@@ -243,40 +243,63 @@ func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (u
 
 		rel, err := c.schema.GetRel(child.Name, ti.Name)
 		if err != nil {
-			skipped |= (1 << uint(id))
+			return 0, nil, err
-			continue
+			//skipped |= (1 << uint(id))
+			//continue
 		}
 
 		switch rel.Type {
 		case RelOneToOne, RelOneToMany:
 			if _, ok := colmap[rel.Right.Col]; !ok {
 				cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Right.Col, FieldName: rel.Right.Col})
-			}
 				colmap[rel.Right.Col] = struct{}{}
+			}
 
 		case RelOneToManyThrough:
 			if _, ok := colmap[rel.Left.Col]; !ok {
 				cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Left.Col})
-			}
 				colmap[rel.Left.Col] = struct{}{}
+			}
+
+		case RelEmbedded:
+			if _, ok := colmap[rel.Left.Col]; !ok {
+				cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Left.Col})
+				colmap[rel.Left.Col] = struct{}{}
+			}
 
 		case RelRemote:
 			if _, ok := colmap[rel.Left.Col]; !ok {
 				cols = append(cols, &qcode.Column{Table: ti.Name, Name: rel.Left.Col, FieldName: rel.Right.Col})
-			}
 				colmap[rel.Left.Col] = struct{}{}
 				skipped |= (1 << uint(id))
+			}
 
 		default:
-			skipped |= (1 << uint(id))
+			return 0, nil, fmt.Errorf("unknown relationship %s", rel)
+			//skipped |= (1 << uint(id))
 		}
 	}
 
-	return skipped, cols
+	return skipped, cols, nil
 }
 
 func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint32, error) {
-	skipped, childCols := c.processChildren(sel, ti)
+	var rel *DBRel
+	var err error
+
+	if sel.ParentID != -1 {
+		parent := c.s[sel.ParentID]
+
+		rel, err = c.schema.GetRel(ti.Name, parent.Name)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	skipped, childCols, err := c.processChildren(sel, ti)
+	if err != nil {
+		return 0, err
+	}
 	hasOrder := len(sel.OrderBy) != 0
 
 	// SELECT
@@ -288,9 +311,8 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint
 		io.WriteString(c.w, `"`)
 
 		if hasOrder {
-			err := c.renderOrderBy(sel, ti)
+			if err := c.renderOrderBy(sel, ti); err != nil {
-			if err != nil {
+				return 0, err
-				return skipped, err
 			}
 		}
 
@@ -319,8 +341,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint
 
 	c.renderRemoteRelColumns(sel, ti)
 
-	err := c.renderJoinedColumns(sel, ti, skipped)
+	if err = c.renderJoinedColumns(sel, ti, skipped); err != nil {
-	if err != nil {
 		return skipped, err
 	}
 
@@ -339,7 +360,7 @@ func (c *compilerContext) renderSelect(sel *qcode.Select, ti *DBTableInfo) (uint
 	// END-SELECT
 
 	// FROM (SELECT .... )
-	err = c.renderBaseSelect(sel, ti, childCols, skipped)
+	err = c.renderBaseSelect(sel, ti, rel, childCols, skipped)
 	if err != nil {
 		return skipped, err
 	}
@@ -527,11 +548,11 @@ func (c *compilerContext) renderJoinedColumns(sel *qcode.Select, ti *DBTableInfo
 	return nil
 }
 
-func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
+func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo, rel *DBRel,
 	childCols []*qcode.Column, skipped uint32) error {
 	var groupBy []int
 
-	isRoot := sel.ParentID == -1
+	isRoot := (rel == nil)
 	isFil := (sel.Where != nil && sel.Where.Op != qcode.OpNop)
 	isSearch := sel.Args["search"] != nil
 	isAgg := false
@@ -682,10 +703,7 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
 
 	io.WriteString(c.w, ` FROM `)
 
-	//fmt.Fprintf(w, ` FROM "%s"`, c.sel.Name)
+	c.renderFrom(sel, ti, rel)
-	io.WriteString(c.w, `"`)
-	io.WriteString(c.w, ti.Name)
-	io.WriteString(c.w, `"`)
 
 	// if tn, ok := c.tmap[sel.Name]; ok {
 	// 	//fmt.Fprintf(w, ` FROM "%s" AS "%s"`, tn, c.sel.Name)
@@ -711,11 +729,9 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
 		}
 
 		io.WriteString(c.w, ` WHERE (`)
-
 		if err := c.renderRelationship(sel, ti); err != nil {
 			return err
 		}
-
 		if isFil {
 			io.WriteString(c.w, ` AND `)
 			if err := c.renderWhere(sel, ti); err != nil {
@@ -770,6 +786,44 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
 	return nil
 }
 
+func (c *compilerContext) renderFrom(sel *qcode.Select, ti *DBTableInfo, rel *DBRel) error {
+	if rel != nil && rel.Type == RelEmbedded {
+		// json_to_recordset('[{"a":1,"b":[1,2,3],"c":"bar"}, {"a":2,"b":[1,2,3],"c":"bar"}]') as x(a int, b text, d text);
+
+		io.WriteString(c.w, `"`)
+		io.WriteString(c.w, rel.Left.Table)
+		io.WriteString(c.w, `", `)
+
+		io.WriteString(c.w, ti.Type)
+		io.WriteString(c.w, `_to_recordset(`)
+		colWithTable(c.w, rel.Left.Table, rel.Right.Col)
+		io.WriteString(c.w, `) AS `)
+
+		io.WriteString(c.w, `"`)
+		io.WriteString(c.w, ti.Name)
+		io.WriteString(c.w, `"`)
+
+		io.WriteString(c.w, `(`)
+		for i, col := range ti.Columns {
+			if i != 0 {
+				io.WriteString(c.w, `, `)
+			}
+			io.WriteString(c.w, col.Name)
+			io.WriteString(c.w, ` `)
+			io.WriteString(c.w, col.Type)
+		}
+		io.WriteString(c.w, `)`)
+
+	} else {
+		//fmt.Fprintf(w, ` FROM "%s"`, c.sel.Name)
+		io.WriteString(c.w, `"`)
+		io.WriteString(c.w, ti.Name)
+		io.WriteString(c.w, `"`)
+	}
+
+	return nil
+}
+
 func (c *compilerContext) renderOrderByColumns(sel *qcode.Select, ti *DBTableInfo) {
 	colsRendered := len(sel.Cols) != 0
 
@@ -852,7 +906,13 @@ func (c *compilerContext) renderRelationshipByName(table, parent string, id int3
 			io.WriteString(c.w, `) = (`)
 			colWithTable(c.w, rel.Through, rel.Right.Col)
 		}
+
+	case RelEmbedded:
+		colWithTable(c.w, rel.Left.Table, rel.Left.Col)
+		io.WriteString(c.w, `) = (`)
+		colWithTableID(c.w, parent, id, rel.Left.Col)
 	}
+
 	io.WriteString(c.w, `))`)
 
 	return nil

psql/query_test.go

@@ -463,6 +463,32 @@ func multiRoot(t *testing.T) {
 	}
 }
 
+func jsonColumnAsTable(t *testing.T) {
+	gql := `query {
+		products {
+			id
+			name
+			tag_count {
+				count
+				tags {
+					name
+				}
+			}
+		}
+	}`
+
+	sql := `SELECT json_object_agg('products', json_0) FROM (SELECT coalesce(json_agg("json_0"), '[]') AS "json_0" FROM (SELECT row_to_json((SELECT "json_row_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name", "tag_count_1_join"."json_1" AS "tag_count") AS "json_row_0")) AS "json_0" FROM (SELECT "products"."id", "products"."name" FROM "products" LIMIT ('20') :: integer) AS "products_0" LEFT OUTER JOIN LATERAL (SELECT row_to_json((SELECT "json_row_1" FROM (SELECT "tag_count_1"."count" AS "count", "tags_2_join"."json_2" AS "tags") AS "json_row_1")) AS "json_1" FROM (SELECT "tag_count"."count", "tag_count"."tag_id" FROM "products", json_to_recordset("products"."tag_count") AS "tag_count"(tag_id bigint, count int) WHERE ((("products"."id") = ("products_0"."id"))) LIMIT ('1') :: integer) AS "tag_count_1" LEFT OUTER JOIN LATERAL (SELECT coalesce(json_agg("json_2"), '[]') AS "json_2" FROM (SELECT row_to_json((SELECT "json_row_2" FROM (SELECT "tags_2"."name" AS "name") AS "json_row_2")) AS "json_2" FROM (SELECT "tags"."name" FROM "tags" WHERE ((("tags"."id") = ("tag_count_1"."tag_id"))) LIMIT ('20') :: integer) AS "tags_2" LIMIT ('20') :: integer) AS "json_agg_2") AS "tags_2_join" ON ('true') LIMIT ('1') :: integer) AS "tag_count_1_join" ON ('true') LIMIT ('20') :: integer) AS "json_agg_0") AS "sel_0"`
+
+	resSQL, err := compileGQLToPSQL(gql, nil, "admin")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(resSQL) != sql {
+		t.Fatal(errNotExpected)
+	}
+}
+
 func skipUserIDForAnonRole(t *testing.T) {
 	gql := `query {
 		products {
@@ -548,6 +574,7 @@ func TestCompileQuery(t *testing.T) {
 	t.Run("queryWithVariables", queryWithVariables)
 	t.Run("withWhereOnRelations", withWhereOnRelations)
 	t.Run("multiRoot", multiRoot)
+	t.Run("jsonColumnAsTable", jsonColumnAsTable)
 	t.Run("skipUserIDForAnonRole", skipUserIDForAnonRole)
 	t.Run("blockedQuery", blockedQuery)
 	t.Run("blockedFunctions", blockedFunctions)

psql/schema.go

@@ -15,6 +15,7 @@ type DBSchema struct {
 
 type DBTableInfo struct {
 	Name       string
+	Type       string
 	Singular   bool
 	Columns    []DBColumn
 	PrimaryCol *DBColumn
@@ -29,6 +30,7 @@ const (
 	RelOneToOne RelType = iota + 1
 	RelOneToMany
 	RelOneToManyThrough
+	RelEmbedded
 	RelRemote
 )
 
@@ -51,7 +53,6 @@ type DBRel struct {
 }
 
 func NewDBSchema(info *DBInfo, aliases map[string][]string) (*DBSchema, error) {
-
 	schema := &DBSchema{
 		t:  make(map[string]*DBTableInfo),
 		rm: make(map[string]map[string]*DBRel),
@@ -83,6 +84,7 @@ func (s *DBSchema) addTable(
 	singular := flect.Singularize(t.Key)
 	s.t[singular] = &DBTableInfo{
 		Name:     t.Name,
+		Type:     t.Type,
 		Singular: true,
 		Columns:  cols,
 		ColMap:   colmap,
@@ -92,6 +94,7 @@ func (s *DBSchema) addTable(
 	plural := flect.Pluralize(t.Key)
 	s.t[plural] = &DBTableInfo{
 		Name:     t.Name,
+		Type:     t.Type,
 		Singular: false,
 		Columns:  cols,
 		ColMap:   colmap,
@@ -139,19 +142,43 @@ func (s *DBSchema) updateRelationships(t DBTable, cols []DBColumn) error {
 	for i := range cols {
 		c := cols[i]
 
-		if len(c.FKeyTable) == 0 || len(c.FKeyColID) == 0 {
+		if len(c.FKeyTable) == 0 {
 			continue
 		}
 
 		// Foreign key column name
 		ft := strings.ToLower(c.FKeyTable)
-		fcid := c.FKeyColID[0]
 
 		ti, ok := s.t[ft]
 		if !ok {
 			return fmt.Errorf("invalid foreign key table '%s'", ft)
 		}
 
+		// This is an embedded relationship like when a json/jsonb column
+		// is exposed as a table
+		if c.Name == c.FKeyTable && len(c.FKeyColID) == 0 {
+			rel := &DBRel{Type: RelEmbedded}
+			rel.Left.col = cti.PrimaryCol
+			rel.Left.Table = cti.Name
+			rel.Left.Col = cti.PrimaryCol.Name
+
+			rel.Right.col = &c
+			rel.Right.Table = ti.Name
+			rel.Right.Col = c.Name
+
+			if err := s.SetRel(ft, ct, rel); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if len(c.FKeyColID) == 0 {
+			continue
+		}
+
+		// Foreign key column id
+		fcid := c.FKeyColID[0]
+
 		fc, ok := ti.ColIDMap[fcid]
 		if !ok {
 			return fmt.Errorf("invalid foreign key column id '%d' for table '%s'",

psql/strings.go

@@ -12,6 +12,8 @@ func (rt RelType) String() string {
 		return "one to many through"
 	case RelRemote:
 		return "remote"
+	case RelEmbedded:
+		return "embedded"
 	}
 	return ""
 }

psql/tables.go

@@ -62,6 +62,20 @@ func GetDBInfo(db *pgxpool.Pool) (*DBInfo, error) {
 	return di, nil
 }
 
+func (di *DBInfo) AddTable(t DBTable, cols []DBColumn) {
+	t.ID = di.Tables[len(di.Tables)-1].ID
+
+	di.Tables = append(di.Tables, t)
+	di.colmap[t.Key] = make(map[string]*DBColumn, len(cols))
+
+	for i := range cols {
+		cols[i].ID = int16(i)
+		c := &cols[i]
+		di.colmap[t.Key][c.Key] = c
+	}
+	di.Columns = append(di.Columns, cols)
+}
+
 func (di *DBInfo) GetColumn(table, column string) (*DBColumn, bool) {
 	v, ok := di.colmap[strings.ToLower(table)][strings.ToLower(column)]
 	return v, ok

serv/config.go

@@ -87,6 +87,7 @@ type config struct {
 
 type configColumn struct {
 	Name       string
+	Type       string
 	ForeignKey string `mapstructure:"related_to"`
 }
 
@@ -313,7 +314,7 @@ func (c *config) getAliasMap() map[string][]string {
 	for i := range c.Tables {
 		t := c.Tables[i]
 
-		if len(t.Table) == 0 {
+		if len(t.Table) == 0 || len(t.Columns) != 0 {
 			continue
 		}
 

serv/config_compile.go

@@ -8,9 +8,61 @@ import (
 	"github.com/dosco/super-graph/qcode"
 )
 
+func addTables(c *config, di *psql.DBInfo) error {
+	for _, t := range c.Tables {
+		if len(t.Table) == 0 || len(t.Columns) == 0 {
+			continue
+		}
+		if err := addTable(di, t.Columns, t); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func addTable(di *psql.DBInfo, cols []configColumn, t configTable) error {
+	bc, ok := di.GetColumn(t.Table, t.Name)
+	if !ok {
+		return fmt.Errorf(
+			"Column '%s' not found on table '%s'",
+			t.Name, t.Table)
+	}
+
+	if bc.Type != "json" && bc.Type != "jsonb" {
+		return fmt.Errorf(
+			"Column '%s' in table '%s' is of type '%s'. Only JSON or JSONB is valid",
+			t.Name, t.Table, bc.Type)
+	}
+
+	table := psql.DBTable{
+		Name: t.Name,
+		Key:  strings.ToLower(t.Name),
+		Type: bc.Type,
+	}
+
+	columns := make([]psql.DBColumn, 0, len(cols))
+
+	for i := range cols {
+		c := cols[i]
+		columns = append(columns, psql.DBColumn{
+			Name: c.Name,
+			Key:  strings.ToLower(c.Name),
+			Type: c.Type,
+		})
+	}
+
+	di.AddTable(table, columns)
+	bc.FKeyTable = t.Name
+
+	return nil
+}
+
 func addForeignKeys(c *config, di *psql.DBInfo) error {
 	for _, t := range c.Tables {
 		for _, c := range t.Columns {
+			if len(c.ForeignKey) == 0 {
+				continue
+			}
 			if err := addForeignKey(di, c, t); err != nil {
 				return err
 			}
@@ -23,7 +75,7 @@ func addForeignKey(di *psql.DBInfo, c configColumn, t configTable) error {
 	c1, ok := di.GetColumn(t.Name, c.Name)
 	if !ok {
 		return fmt.Errorf(
-			"Invalid table '%s' or column '%s in config",
+			"Invalid table '%s' or column '%s' in config",
 			t.Name, c.Name)
 	}
 

serv/serv.go

@@ -21,6 +21,10 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
 		return nil, nil, err
 	}
 
+	if err = addTables(c, di); err != nil {
+		return nil, nil, err
+	}
+
 	if err = addForeignKeys(c, di); err != nil {
 		return nil, nil, err
 	}