wpetit
/
super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix various json parsing and sql generation bugs

This commit is contained in:
Vikram Rangnekar 2020-01-19 03:12:51 -05:00
parent 8097ca3b8f
commit a0b8907c3c
16 changed files with 180 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/.vuepress/components/HomeLayout.vue
View File

@ -137,7 +137,7 @@
</h1>
<div class="text-2xl md:text-3xl">
<small class="text-sm">Download the Docker compose config for the demo</small>
<pre>&#8227; curl -L -o demo.yml https://bit.ly/2mq05lW</pre>
<pre>&#8227; curl -L -o demo.yml https://bit.ly/2FZS0uw</pre>
<small class="text-sm">Setup the demo database</small>
<pre>&#8227; docker-compose -f demo.yml run rails_app rake db:create db:migrate db:seed</pre>

9
jsn/bench.0 Normal file
View File

@ -0,0 +1,9 @@
goos: darwin
goarch: amd64
pkg: github.com/dosco/super-graph/jsn
BenchmarkGet-8 13310 88437 ns/op 3328 B/op 2 allocs/op
BenchmarkFilter-8 182232 6922 ns/op 448 B/op 1 allocs/op
BenchmarkStrip-8 162709 6560 ns/op 224 B/op 1 allocs/op
BenchmarkReplace-8 85846 13996 ns/op 416 B/op 1 allocs/op
PASS
ok github.com/dosco/super-graph/jsn 5.913s

14
jsn/get.go
View File

@ -130,6 +130,20 @@ func Get(b []byte, keys [][]byte) []Field {
n++
}
if state == expectListClose {
loop:
for j := i + 1; j < len(b); j++ {
switch b[j] {
case ' ', '\t', '\n':
continue
case '{':
break loop
}
i = e
break loop
}
}
state = expectKey
e = 0
}

56
jsn/json_test.go
View File

@ -9,11 +9,11 @@ var (
input1 = `
{
"data": {
"test": { "__twitter_id": "ABCD" },
"test_1a": { "__twitter_id": "ABCD" },
"users": [
{
"id": 1,
"full_name": "'Sidney Stroman'",
"full_name": "'Sidney St[1]roman'",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"embed": {
@ -108,7 +108,7 @@ var (
input2 = `
[{
"id": 1,
"full_name": "Sidney Stroman",
"full_name": "Sidney St[1]roman",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"something": null,
@ -130,7 +130,7 @@ var (
input3 = `
{
"data": {
"test": { "__twitter_id": "ABCD" },
"test_1a": { "__twitter_id": "ABCD" },
"users": [{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]
}
}`
@ -138,7 +138,7 @@ var (
input4 = `
{ "users" : [{
"id": 1,
"full_name": "Sidney Stroman",
"full_name": "Sidney St[1]roman",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"embed": {
@ -155,24 +155,26 @@ var (
"email": "user1@demo.com",
"__twitter_id": [{ "name": "hello" }, { "name": "world"}]
}] }`
input5 = `
{"data":{"title":"In September 2018, Slovak police stated that Kuciak was murdered because of his investigative work, and that the murder had been ordered.[9][10] They arrested eight suspects,[11] charging three of them with first-degree murder.[11]","topics":["cpp"]},"a":["1111"]},"thread_slug":"in-september-2018-slovak-police-stated-that-kuciak-7929",}`
)
func TestGet(t *testing.T) {
values := Get([]byte(input1), [][]byte{
[]byte("test_1a"),
[]byte("__twitter_id"),
[]byte("work_email"),
})
expected := []Field{
{[]byte("test_1a"), []byte(`{ "__twitter_id": "ABCD" }`)},
{[]byte("__twitter_id"), []byte(`"ABCD"`)},
{[]byte("__twitter_id"), []byte(`"2048666903444506956"`)},
{[]byte("__twitter_id"), []byte(`"ABC123"`)},
{[]byte("__twitter_id"), []byte(`"more123"`)},
{[]byte("__twitter_id"),
[]byte(`[{ "name": "hello" }, { "name": "world"}]`)},
{[]byte("__twitter_id"),
[]byte(`{ "name": "\"hellos\"", "address": { "work": "1 infinity loop" } }`),
},
{[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)},
{[]byte("__twitter_id"), []byte(`{ "name": "\"hellos\"", "address": { "work": "1 infinity loop" } }`)},
{[]byte("__twitter_id"), []byte(`1234567890`)},
{[]byte("__twitter_id"), []byte(`1.23E`)},
{[]byte("__twitter_id"), []byte(`true`)},
@ -201,6 +203,30 @@ func TestGet(t *testing.T) {
}
}
func TestGet1(t *testing.T) {
values := Get([]byte(input5), [][]byte{
[]byte("thread_slug"),
})
expected := []Field{
{[]byte("thread_slug"), []byte(`"in-september-2018-slovak-police-stated-that-kuciak-7929"`)},
}
if len(values) != len(expected) {
t.Fatal("len(values) != len(expected)")
}
for i := range expected {
if !bytes.Equal(values[i].Key, expected[i].Key) {
t.Error(string(values[i].Key), " != ", string(expected[i].Key))
}
if !bytes.Equal(values[i].Value, expected[i].Value) {
t.Error(string(values[i].Value), " != ", string(expected[i].Value))
}
}
}
func TestValue(t *testing.T) {
v1 := []byte("12345")
if !bytes.Equal(Value(v1), v1) {
@ -230,7 +256,7 @@ func TestFilter1(t *testing.T) {
t.Error(err)
}
expected := `[{"id": 1,"full_name": "Sidney Stroman","embed": {"id": 8,"full_name": "Caroll Orn Sr.","email": "joannarau@hegmann.io","__twitter_id": "ABC123"}},{"id": 2,"full_name": "Jerry Dickinson"}]`
expected := `[{"id": 1,"full_name": "Sidney St[1]roman","embed": {"id": 8,"full_name": "Caroll Orn Sr.","email": "joannarau@hegmann.io","__twitter_id": "ABC123"}},{"id": 2,"full_name": "Jerry Dickinson"}]`
if b.String() != expected {
t.Error("Does not match expected json")
@ -306,7 +332,7 @@ func TestReplace(t *testing.T) {
expected := `{ "users" : [{
"id": 1,
"full_name": "Sidney Stroman",
"full_name": "Sidney St[1]roman",
"email": "user0@demo.com",
"__twitter_id": "2048666903444506956",
"embed": {
@ -338,7 +364,7 @@ func TestReplace(t *testing.T) {
func TestReplaceEmpty(t *testing.T) {
var buf bytes.Buffer
json := `{ "users" : [{"id":1,"full_name":"Sidney Stroman","email":"user0@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":2,"full_name":"Jerry Dickinson","email":"user1@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":3,"full_name":"Kenna Cassin","email":"user2@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":4,"full_name":"Mr. Pat Parisian","email":"rodney@kautzer.biz","__users_twitter_id":"2048666903444506956"}, {"id":5,"full_name":"Bette Ebert","email":"janeenrath@goyette.com","__users_twitter_id":"2048666903444506956"}, {"id":6,"full_name":"Everett Kiehn","email":"michael@bartoletti.com","__users_twitter_id":"2048666903444506956"}, {"id":7,"full_name":"Katrina Cronin","email":"loretaklocko@framivolkman.org","__users_twitter_id":"2048666903444506956"}, {"id":8,"full_name":"Caroll Orn Sr.","email":"joannarau@hegmann.io","__users_twitter_id":"2048666903444506956"}, {"id":9,"full_name":"Gwendolyn Ziemann","email":"renaytoy@rutherford.co","__users_twitter_id":"2048666903444506956"}, {"id":10,"full_name":"Mrs. Rosann Fritsch","email":"holliemosciski@thiel.org","__users_twitter_id":"2048666903444506956"}, {"id":11,"full_name":"Arden Koss","email":"cristobalankunding@howewelch.org","__users_twitter_id":"2048666903444506956"}, {"id":12,"full_name":"Brenton Bauch PhD","email":"renee@miller.co","__users_twitter_id":"2048666903444506956"}, {"id":13,"full_name":"Daine Gleichner","email":"andrea@nienow.co","__users_twitter_id":"2048666903444506956"}] }`
json := `{ "users" : [{"id":1,"full_name":"Sidney St[1]roman","email":"user0@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":2,"full_name":"Jerry Dickinson","email":"user1@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":3,"full_name":"Kenna Cassin","email":"user2@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":4,"full_name":"Mr. Pat Parisian","email":"rodney@kautzer.biz","__users_twitter_id":"2048666903444506956"}, {"id":5,"full_name":"Bette Ebert","email":"janeenrath@goyette.com","__users_twitter_id":"2048666903444506956"}, {"id":6,"full_name":"Everett Kiehn","email":"michael@bartoletti.com","__users_twitter_id":"2048666903444506956"}, {"id":7,"full_name":"Katrina Cronin","email":"loretaklocko@framivolkman.org","__users_twitter_id":"2048666903444506956"}, {"id":8,"full_name":"Caroll Orn Sr.","email":"joannarau@hegmann.io","__users_twitter_id":"2048666903444506956"}, {"id":9,"full_name":"Gwendolyn Ziemann","email":"renaytoy@rutherford.co","__users_twitter_id":"2048666903444506956"}, {"id":10,"full_name":"Mrs. Rosann Fritsch","email":"holliemosciski@thiel.org","__users_twitter_id":"2048666903444506956"}, {"id":11,"full_name":"Arden Koss","email":"cristobalankunding@howewelch.org","__users_twitter_id":"2048666903444506956"}, {"id":12,"full_name":"Brenton Bauch PhD","email":"renee@miller.co","__users_twitter_id":"2048666903444506956"}, {"id":13,"full_name":"Daine Gleichner","email":"andrea@nienow.co","__users_twitter_id":"2048666903444506956"}] }`
err := Replace(&buf, []byte(json), []Field{}, []Field{})
if err != nil {
@ -395,7 +421,7 @@ func TestKeys3(t *testing.T) {
json := `{
"insert": {
"created_at": "now",
"test": { "type1": "a", "type2": "b" },
"test_1a": { "type1": "a", "type2": "b" },
"name": "Hello",
"updated_at": "now",
"description": "World"
@ -406,7 +432,7 @@ func TestKeys3(t *testing.T) {
fields := Keys([]byte(json))
exp := []string{
"insert", "created_at", "test", "type1", "type2", "name", "updated_at", "description",
"insert", "created_at", "test_1a", "type1", "type2", "name", "updated_at", "description",
"user",
}

13
jsn/keys.go
View File

@ -111,6 +111,19 @@ func Keys(b []byte) [][]byte {
res = append(res, k)
}
if state == expectListClose {
loop:
for j := i + 1; j < len(b); j++ {
switch b[j] {
case ' ', '\t', '\n':
continue
case '{':
break loop
}
i = e
break loop
}
}
state = expectKey
k = nil
e = 0

27
psql/query.go
View File

@ -217,6 +217,10 @@ func (c *compilerContext) processChildren(sel *qcode.Select, ti *DBTableInfo) (u
colmap[sel.Cols[i].Name] = struct{}{}
}
for i := range sel.OrderBy {
colmap[sel.OrderBy[i].Col] = struct{}{}
}
for _, id := range sel.Children {
child := &c.s[id]
@ -510,11 +514,14 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
isSearch := sel.Args["search"] != nil
isAgg := false
colmap := make(map[string]struct{}, (len(sel.Cols) + len(sel.OrderBy)))
io.WriteString(c.w, ` FROM (SELECT `)
i := 0
for n, col := range sel.Cols {
cn := col.Name
colmap[cn] = struct{}{}
_, isRealCol := ti.ColMap[cn]
@ -625,7 +632,27 @@ func (c *compilerContext) renderBaseSelect(sel *qcode.Select, ti *DBTableInfo,
}
}
// if i != 0 && len(sel.OrderBy) != 0 {
// io.WriteString(c.w, ", ")
// }
for _, ob := range sel.OrderBy {
if _, ok := colmap[ob.Col]; ok {
continue
}
colmap[ob.Col] = struct{}{}
if i != 0 {
io.WriteString(c.w, `, `)
}
colWithTable(c.w, ti.Name, ob.Col)
i++
}
for _, col := range childCols {
if _, ok := colmap[col.Name]; ok {
continue
}
if i != 0 {
io.WriteString(c.w, `, `)
}

8
serv/allow.go
View File

@ -128,10 +128,10 @@ func (al *allowList) upsert(query, vars []byte, uri string) {
var key string
if len(name) == 0 {
key = hash
} else {
if len(name) != 0 {
key = name
} else {
key = hash
}
if i, ok := al.index[key]; !ok {
@ -280,7 +280,7 @@ func (al *allowList) save(item *allowItem) {
for i := range v {
if len(v[i].vars) != 0 && !bytes.Equal(v[i].vars, []byte("{}")) {
vj, err := json.MarshalIndent(v[i].vars, "", "\t")
vj, err := json.MarshalIndent(v[i].vars, "", " ")
if err != nil {
logger.Warn().Err(err).Msg("Failed to write allow list 'vars' to file")
continue

2
serv/args.go
View File

@ -35,6 +35,8 @@ func argMap(ctx context.Context, vars []byte) func(w io.Writer, tag string) (int
fields := jsn.Get(vars, [][]byte{[]byte(tag)})
fmt.Println(">>", tag, string(vars))
if len(fields) == 0 {
return 0, nil
}

2
serv/cmd.go
View File

@ -324,7 +324,7 @@ Branch : %v
Go version : %v
Licensed under the Apache Public License 2.0
Copyright 2015-2019 Vikram Rangnekar.
Copyright 2020, Vikram Rangnekar.
`,
version,
lastCommitSHA,

4
serv/cmd_migrate.go
View File

@ -63,7 +63,7 @@ func cmdDBCreate(cmd *cobra.Command, args []string) {
}
defer conn.Close(ctx)
sql := fmt.Sprintf("CREATE DATABASE %s", conf.DB.DBName)
sql := fmt.Sprintf(`CREATE DATABASE "%s"`, conf.DB.DBName)
_, err = conn.Exec(ctx, sql)
if err != nil {
@ -83,7 +83,7 @@ func cmdDBDrop(cmd *cobra.Command, args []string) {
}
defer conn.Close(ctx)
sql := fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, conf.DB.DBName)
sql := fmt.Sprintf(`DROP DATABASE IF EXISTS "%s"`, conf.DB.DBName)
_, err = conn.Exec(ctx, sql)
if err != nil {

10
serv/cmd_serv.go
View File

@ -8,17 +8,19 @@ func cmdServ(cmd *cobra.Command, args []string) {
var err error
if conf, err = initConf(); err != nil {
errlog.Fatal().Err(err).Msg("failed to read config")
fatalInProd(err, "failed to read config")
}
db, err = initDBPool(conf)
if err != nil {
errlog.Fatal().Err(err).Msg("failed to connect to database")
if conf != nil {
if db, err = initDBPool(conf); err != nil {
fatalInProd(err, "failed to connect to database")
}
initCompiler()
initAllowList(confPath)
initPreparedList()
}
initWatcher(confPath)
startHTTP()

2
serv/reload.go
View File

@ -108,7 +108,7 @@ func Do(log func(string, ...interface{}), additional ...dir) error {
// Ensure that we use the correct events, as they are not uniform across
// platforms. See https://github.com/fsnotify/fsnotify/issues/74
if !conf.Production && strings.HasSuffix(event.Name, "/allow.list") {
if conf != nil && !conf.Production && strings.HasSuffix(event.Name, "/allow.list") {
continue
}

32
serv/serv.go
View File

@ -50,7 +50,7 @@ func initCompilers(c *config) (*qcode.Compiler, *psql.Compiler, error) {
}
func initWatcher(cpath string) {
if !conf.WatchAndReload {
if conf != nil && !conf.WatchAndReload {
return
}
@ -70,8 +70,17 @@ func initWatcher(cpath string) {
}
func startHTTP() {
var hostPort string
var appName string
defaultHP := "0.0.0.0:8080"
env := os.Getenv("GO_ENV")
if conf != nil {
appName = conf.AppName
hp := strings.SplitN(conf.HostPort, ":", 2)
if len(hp) == 2 {
if len(conf.Host) != 0 {
hp[0] = conf.Host
}
@ -80,7 +89,13 @@ func startHTTP() {
hp[1] = conf.Port
}
hostPort := fmt.Sprintf("%s:%s", hp[0], hp[1])
hostPort = fmt.Sprintf("%s:%s", hp[0], hp[1])
}
}
if len(hostPort) == 0 {
hostPort = defaultHP
}
srv := &http.Server{
Addr: hostPort,
@ -110,8 +125,8 @@ func startHTTP() {
Str("version", version).
Str("git_branch", gitBranch).
Str("host_post", hostPort).
Str("app_name", conf.AppName).
Str("env", conf.Env).
Str("app_name", appName).
Str("env", env).
Msgf("%s listening", serverName)
if err := srv.ListenAndServe(); err != http.ErrServerClosed {
@ -124,7 +139,7 @@ func startHTTP() {
func routeHandler() http.Handler {
var apiH http.Handler
if conf.HTTPGZip {
if conf != nil && conf.HTTPGZip {
gzipH := gziphandler.MustNewGzipLevelHandler(6)
apiH = gzipH(http.HandlerFunc(apiV1))
} else {
@ -132,12 +147,15 @@ func routeHandler() http.Handler {
}
mux := http.NewServeMux()
if conf != nil {
mux.HandleFunc("/health", health)
mux.Handle("/api/v1/graphql", withAuth(apiH))
if conf.WebUI {
mux.Handle("/", http.FileServer(rice.MustFindBox("../web/build").HTTPBox()))
}
}
fn := func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Server", serverName)
@ -170,3 +188,7 @@ func getConfigName() string {
return ge
}
func isDev() bool {
return strings.HasPrefix(os.Getenv("GO_ENV"), "dev")
}

8
serv/utils.go
View File

@ -141,3 +141,11 @@ func findStmt(role string, stmts []stmt) *stmt {
}
return nil
}
func fatalInProd(err error, msg string) {
if isDev() {
errlog.Error().Err(err).Msg(msg)
} else {
errlog.Fatal().Err(err).Msg(msg)
}
}