Fix various json parsing and sql generation bugs

docs/.vuepress/components/HomeLayout.vue

@@ -137,7 +137,7 @@
           </h1>
           <div class="text-2xl md:text-3xl">
             <small class="text-sm">Download the Docker compose config for the demo</small>
-            <pre>&#8227; curl -L -o demo.yml https://bit.ly/2mq05lW</pre>
+            <pre>&#8227; curl -L -o demo.yml https://bit.ly/2FZS0uw</pre>
 
             <small class="text-sm">Setup the demo database</small>
             <pre>&#8227; docker-compose -f demo.yml run rails_app rake db:create db:migrate db:seed</pre>

jsn/bench.0

@@ -0,0 +1,9 @@
+goos: darwin
+goarch: amd64
+pkg: github.com/dosco/super-graph/jsn
+BenchmarkGet-8       	   13310	     88437 ns/op	    3328 B/op	       2 allocs/op
+BenchmarkFilter-8    	  182232	      6922 ns/op	     448 B/op	       1 allocs/op
+BenchmarkStrip-8     	  162709	      6560 ns/op	     224 B/op	       1 allocs/op
+BenchmarkReplace-8   	   85846	     13996 ns/op	     416 B/op	       1 allocs/op
+PASS
+ok  	github.com/dosco/super-graph/jsn	5.913s

jsn/get.go

@@ -130,6 +130,20 @@ func Get(b []byte, keys [][]byte) []Field {
 				n++
 			}
 
+			if state == expectListClose {
+			loop:
+				for j := i + 1; j < len(b); j++ {
+					switch b[j] {
+					case ' ', '\t', '\n':
+						continue
+					case '{':
+						break loop
+					}
+					i = e
+					break loop
+				}
+			}
+
 			state = expectKey
 			e = 0
 		}

jsn/json_test.go

@@ -9,11 +9,11 @@ var (
 	input1 = `
 	{ 
 	"data": {
-	"test": { "__twitter_id": "ABCD" },
+	"test_1a": { "__twitter_id": "ABCD" },
 	"users": [
 		{
 			"id": 1,
-			"full_name": "'Sidney Stroman'",
+			"full_name": "'Sidney St[1]roman'",
 			"email": "user0@demo.com",
 			"__twitter_id": "2048666903444506956",
 			"embed": {
@@ -108,7 +108,7 @@ var (
 	input2 = `
 	[{
 		"id": 1,
-		"full_name": "Sidney Stroman",
+		"full_name": "Sidney St[1]roman",
 		"email": "user0@demo.com",
 		"__twitter_id": "2048666903444506956",
 		"something": null,
@@ -130,7 +130,7 @@ var (
 	input3 = `
 	{ 
 		"data": {
-			"test": { "__twitter_id": "ABCD" },
+			"test_1a": { "__twitter_id": "ABCD" },
 			"users": [{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]
 		}
 	}`
@@ -138,7 +138,7 @@ var (
 	input4 = `
 	{ "users" : [{
 		"id": 1,
-		"full_name": "Sidney Stroman",
+		"full_name": "Sidney St[1]roman",
 		"email": "user0@demo.com",
 		"__twitter_id": "2048666903444506956",
 		"embed": {
@@ -155,24 +155,26 @@ var (
 		"email": "user1@demo.com",
 		"__twitter_id": [{ "name": "hello" }, { "name": "world"}]
 	}] }`
+
+	input5 = `
+	{"data":{"title":"In September 2018, Slovak police stated that Kuciak was murdered because of his investigative work, and that the murder had been ordered.[9][10] They arrested eight suspects,[11] charging three of them with first-degree murder.[11]","topics":["cpp"]},"a":["1111"]},"thread_slug":"in-september-2018-slovak-police-stated-that-kuciak-7929",}`
 )
 
 func TestGet(t *testing.T) {
 	values := Get([]byte(input1), [][]byte{
+		[]byte("test_1a"),
 		[]byte("__twitter_id"),
 		[]byte("work_email"),
 	})
 
 	expected := []Field{
+		{[]byte("test_1a"), []byte(`{ "__twitter_id": "ABCD" }`)},
 		{[]byte("__twitter_id"), []byte(`"ABCD"`)},
 		{[]byte("__twitter_id"), []byte(`"2048666903444506956"`)},
 		{[]byte("__twitter_id"), []byte(`"ABC123"`)},
 		{[]byte("__twitter_id"), []byte(`"more123"`)},
-		{[]byte("__twitter_id"),
+		{[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)},
-			[]byte(`[{ "name": "hello" }, { "name": "world"}]`)},
+		{[]byte("__twitter_id"), []byte(`{ "name": "\"hellos\"", "address": { "work": "1 infinity loop" } }`)},
-		{[]byte("__twitter_id"),
-			[]byte(`{ "name": "\"hellos\"", "address": { "work": "1 infinity loop" } }`),
-		},
 		{[]byte("__twitter_id"), []byte(`1234567890`)},
 		{[]byte("__twitter_id"), []byte(`1.23E`)},
 		{[]byte("__twitter_id"), []byte(`true`)},
@@ -201,6 +203,30 @@ func TestGet(t *testing.T) {
 	}
 }
 
+func TestGet1(t *testing.T) {
+	values := Get([]byte(input5), [][]byte{
+		[]byte("thread_slug"),
+	})
+
+	expected := []Field{
+		{[]byte("thread_slug"), []byte(`"in-september-2018-slovak-police-stated-that-kuciak-7929"`)},
+	}
+
+	if len(values) != len(expected) {
+		t.Fatal("len(values) != len(expected)")
+	}
+
+	for i := range expected {
+		if !bytes.Equal(values[i].Key, expected[i].Key) {
+			t.Error(string(values[i].Key), " != ", string(expected[i].Key))
+		}
+
+		if !bytes.Equal(values[i].Value, expected[i].Value) {
+			t.Error(string(values[i].Value), " != ", string(expected[i].Value))
+		}
+	}
+}
+
 func TestValue(t *testing.T) {
 	v1 := []byte("12345")
 	if !bytes.Equal(Value(v1), v1) {
@@ -230,7 +256,7 @@ func TestFilter1(t *testing.T) {
 		t.Error(err)
 	}
 
-	expected := `[{"id": 1,"full_name": "Sidney Stroman","embed": {"id": 8,"full_name": "Caroll Orn Sr.","email": "joannarau@hegmann.io","__twitter_id": "ABC123"}},{"id": 2,"full_name": "Jerry Dickinson"}]`
+	expected := `[{"id": 1,"full_name": "Sidney St[1]roman","embed": {"id": 8,"full_name": "Caroll Orn Sr.","email": "joannarau@hegmann.io","__twitter_id": "ABC123"}},{"id": 2,"full_name": "Jerry Dickinson"}]`
 
 	if b.String() != expected {
 		t.Error("Does not match expected json")
@@ -306,7 +332,7 @@ func TestReplace(t *testing.T) {
 
 	expected := `{ "users" : [{
 		"id": 1,
-		"full_name": "Sidney Stroman",
+		"full_name": "Sidney St[1]roman",
 		"email": "user0@demo.com",
 		"__twitter_id": "2048666903444506956",
 		"embed": {
@@ -338,7 +364,7 @@ func TestReplace(t *testing.T) {
 func TestReplaceEmpty(t *testing.T) {
 	var buf bytes.Buffer
 
-	json := `{ "users" : [{"id":1,"full_name":"Sidney Stroman","email":"user0@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":2,"full_name":"Jerry Dickinson","email":"user1@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":3,"full_name":"Kenna Cassin","email":"user2@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":4,"full_name":"Mr. Pat Parisian","email":"rodney@kautzer.biz","__users_twitter_id":"2048666903444506956"}, {"id":5,"full_name":"Bette Ebert","email":"janeenrath@goyette.com","__users_twitter_id":"2048666903444506956"}, {"id":6,"full_name":"Everett Kiehn","email":"michael@bartoletti.com","__users_twitter_id":"2048666903444506956"}, {"id":7,"full_name":"Katrina Cronin","email":"loretaklocko@framivolkman.org","__users_twitter_id":"2048666903444506956"}, {"id":8,"full_name":"Caroll Orn Sr.","email":"joannarau@hegmann.io","__users_twitter_id":"2048666903444506956"}, {"id":9,"full_name":"Gwendolyn Ziemann","email":"renaytoy@rutherford.co","__users_twitter_id":"2048666903444506956"}, {"id":10,"full_name":"Mrs. Rosann Fritsch","email":"holliemosciski@thiel.org","__users_twitter_id":"2048666903444506956"}, {"id":11,"full_name":"Arden Koss","email":"cristobalankunding@howewelch.org","__users_twitter_id":"2048666903444506956"}, {"id":12,"full_name":"Brenton Bauch PhD","email":"renee@miller.co","__users_twitter_id":"2048666903444506956"}, {"id":13,"full_name":"Daine Gleichner","email":"andrea@nienow.co","__users_twitter_id":"2048666903444506956"}] }`
+	json := `{ "users" : [{"id":1,"full_name":"Sidney St[1]roman","email":"user0@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":2,"full_name":"Jerry Dickinson","email":"user1@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":3,"full_name":"Kenna Cassin","email":"user2@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":4,"full_name":"Mr. Pat Parisian","email":"rodney@kautzer.biz","__users_twitter_id":"2048666903444506956"}, {"id":5,"full_name":"Bette Ebert","email":"janeenrath@goyette.com","__users_twitter_id":"2048666903444506956"}, {"id":6,"full_name":"Everett Kiehn","email":"michael@bartoletti.com","__users_twitter_id":"2048666903444506956"}, {"id":7,"full_name":"Katrina Cronin","email":"loretaklocko@framivolkman.org","__users_twitter_id":"2048666903444506956"}, {"id":8,"full_name":"Caroll Orn Sr.","email":"joannarau@hegmann.io","__users_twitter_id":"2048666903444506956"}, {"id":9,"full_name":"Gwendolyn Ziemann","email":"renaytoy@rutherford.co","__users_twitter_id":"2048666903444506956"}, {"id":10,"full_name":"Mrs. Rosann Fritsch","email":"holliemosciski@thiel.org","__users_twitter_id":"2048666903444506956"}, {"id":11,"full_name":"Arden Koss","email":"cristobalankunding@howewelch.org","__users_twitter_id":"2048666903444506956"}, {"id":12,"full_name":"Brenton Bauch PhD","email":"renee@miller.co","__users_twitter_id":"2048666903444506956"}, {"id":13,"full_name":"Daine Gleichner","email":"andrea@nienow.co","__users_twitter_id":"2048666903444506956"}] }`
 
 	err := Replace(&buf, []byte(json), []Field{}, []Field{})
 	if err != nil {
@@ -395,7 +421,7 @@ func TestKeys3(t *testing.T) {
 	json := `{
 		"insert": {
 			"created_at": "now",
-			"test": { "type1": "a", "type2": "b" },
+			"test_1a": { "type1": "a", "type2": "b" },
 			"name": "Hello",
 			"updated_at": "now",
 			"description": "World"
@@ -406,7 +432,7 @@ func TestKeys3(t *testing.T) {
 	fields := Keys([]byte(json))
 
 	exp := []string{
-		"insert", "created_at", "test", "type1", "type2", "name", "updated_at", "description",
+		"insert", "created_at", "test_1a", "type1", "type2", "name", "updated_at", "description",
 		"user",
 	}
 

jsn/keys.go

@@ -111,6 +111,19 @@ func Keys(b []byte) [][]byte {
 				res = append(res, k)
 			}
 
+			if state == expectListClose {
+			loop:
+				for j := i + 1; j < len(b); j++ {
+					switch b[j] {
+					case ' ', '\t', '\n':
+						continue
+					case '{':
+						break loop
+					}
+					i = e
+					break loop
+				}
+			}
 			state = expectKey
 			k = nil
 			e = 0

psql/query.go

@@ -217,6 +217,10 @@ func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (u
 		colmap[sel.Cols[i].Name] = struct{}{}
 	}
 
+	for i := range sel.OrderBy {
+		colmap[sel.OrderBy[i].Col] = struct{}{}
+	}
+
 	for _, id := range sel.Children {
 		child := &c.s[id]
 
@@ -510,11 +514,14 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
 	isSearch := sel.Args["search"] != nil
 	isAgg := false
 
+	colmap := make(map[string]struct{}, (len(sel.Cols) + len(sel.OrderBy)))
+
 	io.WriteString(c.w, ` FROM (SELECT `)
 
 	i := 0
 	for n, col := range sel.Cols {
 		cn := col.Name
+		colmap[cn] = struct{}{}
 
 		_, isRealCol := ti.ColMap[cn]
 
@@ -625,7 +632,27 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
 		}
 	}
 
+	// if i != 0 && len(sel.OrderBy) != 0 {
+	// 	io.WriteString(c.w, ", ")
+	// }
+
+	for _, ob := range sel.OrderBy {
+		if _, ok := colmap[ob.Col]; ok {
+			continue
+		}
+		colmap[ob.Col] = struct{}{}
+
+		if i != 0 {
+			io.WriteString(c.w, `, `)
+		}
+		colWithTable(c.w, ti.Name, ob.Col)
+		i++
+	}
+
 	for _, col := range childCols {
+		if _, ok := colmap[col.Name]; ok {
+			continue
+		}
 		if i != 0 {
 			io.WriteString(c.w, `, `)
 		}

serv/allow.go

@@ -128,10 +128,10 @@ func (al *allowList) upsert(query, vars []byte, uri string) {
 
 	var key string
 
-	if len(name) == 0 {
+	if len(name) != 0 {
-		key = hash
-	} else {
 		key = name
+	} else {
+		key = hash
 	}
 
 	if i, ok := al.index[key]; !ok {
@@ -280,7 +280,7 @@ func (al *allowList) save(item *allowItem) {
 
 		for i := range v {
 			if len(v[i].vars) != 0 && !bytes.Equal(v[i].vars, []byte("{}")) {
-				vj, err := json.MarshalIndent(v[i].vars, "", "\t")
+				vj, err := json.MarshalIndent(v[i].vars, "", "  ")
 				if err != nil {
 					logger.Warn().Err(err).Msg("Failed to write allow list 'vars' to file")
 					continue

serv/args.go

@@ -35,6 +35,8 @@ func argMap(ctx context.Context, vars []byte) func(w io.Writer, tag string) (int
 
 		fields := jsn.Get(vars, [][]byte{[]byte(tag)})
 
+		fmt.Println(">>", tag, string(vars))
+
 		if len(fields) == 0 {
 			return 0, nil
 		}

serv/cmd.go

@@ -324,7 +324,7 @@ Branch                : %v
 Go version            : %v
 
 Licensed under the Apache Public License 2.0
-Copyright 2015-2019 Vikram Rangnekar.
+Copyright 2020, Vikram Rangnekar.
 `,
 		version,
 		lastCommitSHA,

serv/cmd_migrate.go

@@ -63,7 +63,7 @@ func cmdDBCreate(cmd *cobra.Command, args []string) {
 	}
 	defer conn.Close(ctx)
 
-	sql := fmt.Sprintf("CREATE DATABASE %s", conf.DB.DBName)
+	sql := fmt.Sprintf(`CREATE DATABASE "%s"`, conf.DB.DBName)
 
 	_, err = conn.Exec(ctx, sql)
 	if err != nil {
@@ -83,7 +83,7 @@ func cmdDBDrop(cmd *cobra.Command, args []string) {
 	}
 	defer conn.Close(ctx)
 
-	sql := fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, conf.DB.DBName)
+	sql := fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, conf.DB.DBName)
 
 	_, err = conn.Exec(ctx, sql)
 	if err != nil {

serv/cmd_serv.go

@@ -8,17 +8,19 @@ func cmdServ(cmd *cobra.Command, args []string) {
 	var err error
 
 	if conf, err = initConf(); err != nil {
-		errlog.Fatal().Err(err).Msg("failed to read config")
+		fatalInProd(err, "failed to read config")
 	}
 
-	db, err = initDBPool(conf)
+	if conf != nil {
-	if err != nil {
+		if db, err = initDBPool(conf); err != nil {
-		errlog.Fatal().Err(err).Msg("failed to connect to database")
+			fatalInProd(err, "failed to connect to database")
 		}
 
 		initCompiler()
 		initAllowList(confPath)
 		initPreparedList()
+	}
+
 	initWatcher(confPath)
 
 	startHTTP()

serv/reload.go

@@ -108,7 +108,7 @@ func Do(log func(string, ...interface{}), additional ...dir) error {
 				// Ensure that we use the correct events, as they are not uniform across
 				// platforms. See https://github.com/fsnotify/fsnotify/issues/74
 
-				if !conf.Production && strings.HasSuffix(event.Name, "/allow.list") {
+				if conf != nil && !conf.Production && strings.HasSuffix(event.Name, "/allow.list") {
 					continue
 				}
 

serv/serv.go

@@ -50,7 +50,7 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
 }
 
 func initWatcher(cpath string) {
-	if !conf.WatchAndReload {
+	if conf != nil && !conf.WatchAndReload {
 		return
 	}
 
@@ -70,8 +70,17 @@ func initWatcher(cpath string) {
 }
 
 func startHTTP() {
+	var hostPort string
+	var appName string
+
+	defaultHP := "0.0.0.0:8080"
+	env := os.Getenv("GO_ENV")
+
+	if conf != nil {
+		appName = conf.AppName
 		hp := strings.SplitN(conf.HostPort, ":", 2)
 
+		if len(hp) == 2 {
 			if len(conf.Host) != 0 {
 				hp[0] = conf.Host
 			}
@@ -80,7 +89,13 @@ func startHTTP() {
 				hp[1] = conf.Port
 			}
 
-	hostPort := fmt.Sprintf("%s:%s", hp[0], hp[1])
+			hostPort = fmt.Sprintf("%s:%s", hp[0], hp[1])
+		}
+	}
+
+	if len(hostPort) == 0 {
+		hostPort = defaultHP
+	}
 
 	srv := &http.Server{
 		Addr:           hostPort,
@@ -110,8 +125,8 @@ func startHTTP() {
 		Str("version", version).
 		Str("git_branch", gitBranch).
 		Str("host_post", hostPort).
-		Str("app_name", conf.AppName).
+		Str("app_name", appName).
-		Str("env", conf.Env).
+		Str("env", env).
 		Msgf("%s listening", serverName)
 
 	if err := srv.ListenAndServe(); err != http.ErrServerClosed {
@@ -124,7 +139,7 @@ func startHTTP() {
 func routeHandler() http.Handler {
 	var apiH http.Handler
 
-	if conf.HTTPGZip {
+	if conf != nil && conf.HTTPGZip {
 		gzipH := gziphandler.MustNewGzipLevelHandler(6)
 		apiH = gzipH(http.HandlerFunc(apiV1))
 	} else {
@@ -132,12 +147,15 @@ func routeHandler() http.Handler {
 	}
 
 	mux := http.NewServeMux()
+
+	if conf != nil {
 		mux.HandleFunc("/health", health)
 		mux.Handle("/api/v1/graphql", withAuth(apiH))
 
 		if conf.WebUI {
 			mux.Handle("/", http.FileServer(rice.MustFindBox("../web/build").HTTPBox()))
 		}
+	}
 
 	fn := func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Server", serverName)
@@ -170,3 +188,7 @@ func getConfigName() string {
 
 	return ge
 }
+
+func isDev() bool {
+	return strings.HasPrefix(os.Getenv("GO_ENV"), "dev")
+}

serv/utils.go

@@ -141,3 +141,11 @@ func findStmt(role string, stmts []stmt) *stmt {
 	}
 	return nil
 }
+
+func fatalInProd(err error, msg string) {
+	if isDev() {
+		errlog.Error().Err(err).Msg(msg)
+	} else {
+		errlog.Fatal().Err(err).Msg(msg)
+	}
+}