From b66687606496c86a50d1c5144c0167dd69824b0a Mon Sep 17 00:00:00 2001 From: Vikram Rangnekar Date: Sat, 20 Apr 2019 00:35:57 -0400 Subject: [PATCH] Add SQL query cacheing --- config/dev.yml | 2 ++ config/prod.yml | 2 ++ corpus/5 | 2 +- corpus/6 | 17 +--------- docs/guide.md | 29 ++++++++++++++-- go.mod | 1 + go.sum | 2 ++ psql/psql_test.go | 21 ++++++++++++ serv/core.go | 84 +++++++++++++++++++++++++++++++++-------------- 9 files changed, 117 insertions(+), 43 deletions(-) diff --git a/config/dev.yml b/config/dev.yml index 51d7506..64a316a 100644 --- a/config/dev.yml +++ b/config/dev.yml @@ -2,6 +2,8 @@ app_name: "Super Graph Development" host_port: 0.0.0.0:8080 web_ui: true debug_level: 1 + +# enabling tracing also disables query caching enable_tracing: true # Throw a 401 on auth failure for queries that need auth diff --git a/config/prod.yml b/config/prod.yml index 8588175..3b26efa 100644 --- a/config/prod.yml +++ b/config/prod.yml @@ -2,6 +2,8 @@ app_name: "Super Graph Production" host_port: 0.0.0.0:8080 web_ui: false debug_level: 0 + +# disabled tracing enables query caching enable_tracing: false # Throw a 401 on auth failure for queries that need auth diff --git a/corpus/5 b/corpus/5 index 61c166d..02ab731 100644 --- a/corpus/5 +++ b/corpus/5 @@ -1,5 +1,5 @@ query { - product(id: 15) { + product(id: $PRODUCT_ID, where: { price: { eq: $PRODUCT_PRICE } }) { id name } diff --git a/corpus/6 b/corpus/6 index ae381f8..61c166d 100644 --- a/corpus/6 +++ b/corpus/6 @@ -1,21 +1,6 @@ query { - products( - # returns only 30 items - limit: 30, - - # starts from item 10, commented out for now - # offset: 10, - - # orders the response items by highest price - order_by: { price: desc }, - - # no duplicate prices returned - distinct: [ price ] - - # only items with an id >= 30 and < 30 are returned - where: { id: { and: { greater_or_equals: 20, lt: 28 } } }) { + product(id: 15) { id name - price } } \ No newline at end of file diff --git a/docs/guide.md b/docs/guide.md index 87fa899..d89a1f6 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -240,6 +240,29 @@ query { } ``` +### Using variables + +Variables and their values can be passed along side the GraphQL query. Using variables makes for better client side code as well as improved server side +caching. The build-in web ui also hass support for variables. Not having to manupilate your GraphQL query string to insert values into it makes for cleaner +and better client side code. + +```javascript +// define the request object keeping the query and the variables seperate +var req = { + query: '{ product(id: $product_id) { name } }' , + variables: { "product_id": 5 } +} + +// use the fetch api to make the query +fetch('http://localhost:8080/api/v1/graphql', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(req), +}) +.then(res => res.json()) +.then(res => console.log(res.data)); +``` + ### Full text search Every app these days needs search. Enought his often means reaching for something heavy like Solr. While this will work why add complexity to your infrastructure when Postgres has really great @@ -412,11 +435,13 @@ Configuration files can either be in YAML or JSON their names are derived from t We're tried to ensure that the config file is self documenting and easy to work with. ```yaml -title: Super Graph Development +app_name: "Super Graph Development" host_port: 0.0.0.0:8080 web_ui: true debug_level: 1 -enable_tracing: false + +# enabling tracing also disables query caching +enable_tracing: true # Throw a 401 on auth failure for queries that need auth # valid values: always, per_query, never diff --git a/go.mod b/go.mod index 2637978..b24cbf2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/dosco/super-graph require ( github.com/Masterminds/semver v1.4.2 github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3 + github.com/allegro/bigcache v1.2.0 github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/garyburd/redigo v1.6.0 diff --git a/go.sum b/go.sum index e83b729..2449cee 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITg github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3 h1:+qz9Ga6l6lKw6fgvk5RMV5HQznSLvI8Zxajwdj4FhFg= github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3/go.mod h1:FlkD11RtgMTYjVuBnb7cxoHmQGqvPpCsr2atC88nl/M= +github.com/allegro/bigcache v1.2.0 h1:qDaE0QoF29wKBb3+pXFrJFy1ihe5OT9OiXhg1t85SxM= +github.com/allegro/bigcache v1.2.0/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737 h1:rRISKWyXfVxvoa702s91Zl5oREZTrR3yv+tXrrX7G/g= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= diff --git a/psql/psql_test.go b/psql/psql_test.go index a409956..79cf867 100644 --- a/psql/psql_test.go +++ b/psql/psql_test.go @@ -425,6 +425,26 @@ func aggFunctionWithFilter(t *testing.T) { } } +func queryWithVariables(t *testing.T) { + gql := `query { + product(id: $PRODUCT_ID, where: { price: { eq: $PRODUCT_PRICE } }) { + id + name + } + }` + + sql := `SELECT json_object_agg('product', products) FROM (SELECT row_to_json((SELECT "sel_0" FROM (SELECT "products_0"."id" AS "id", "products_0"."name" AS "name") AS "sel_0")) AS "products" FROM (SELECT "products"."id", "products"."name" FROM "products" WHERE ((("products"."price") > (0)) AND (("products"."price") < (8)) AND (("products"."price") = ('{{PRODUCT_PRICE}}')) AND (("id") = ('{{PRODUCT_ID}}'))) LIMIT ('1') :: integer) AS "products_0" LIMIT ('1') :: integer) AS "done_1337";` + + resSQL, err := compileGQLToPSQL(gql) + if err != nil { + t.Fatal(err) + } + + if resSQL != sql { + t.Fatal(errNotExpected) + } +} + func syntheticTables(t *testing.T) { gql := `query { me { @@ -457,6 +477,7 @@ func TestCompileGQL(t *testing.T) { t.Run("manyToManyReverse", manyToManyReverse) t.Run("aggFunction", aggFunction) t.Run("aggFunctionWithFilter", aggFunctionWithFilter) + t.Run("queryWithVariables", queryWithVariables) t.Run("syntheticTables", syntheticTables) } diff --git a/serv/core.go b/serv/core.go index a26d6ce..5167a12 100644 --- a/serv/core.go +++ b/serv/core.go @@ -2,44 +2,74 @@ package serv import ( "context" + "crypto/sha1" "encoding/json" "fmt" "io" "strings" "time" + "github.com/allegro/bigcache" + "github.com/dosco/super-graph/qcode" "github.com/go-pg/pg" "github.com/valyala/fasttemplate" ) +var ( + cache, _ = bigcache.NewBigCache(bigcache.DefaultConfig(24 * time.Hour)) +) + func handleReq(ctx context.Context, w io.Writer, req *gqlReq) error { - qc, err := qcompile.CompileQuery(req.Query) - if err != nil { + var key, finalSQL string + var qc *qcode.QCode + + var entry []byte + var err error + + cacheEnabled := (conf.EnableTracing == false) + + if cacheEnabled { + k := sha1.Sum([]byte(req.Query)) + key = string(k[:]) + entry, err = cache.Get(key) + } + + if len(entry) == 0 || err == bigcache.ErrEntryNotFound { + qc, err = qcompile.CompileQuery(req.Query) + if err != nil { + return err + } + + var sqlStmt strings.Builder + + if err := pcompile.Compile(&sqlStmt, qc); err != nil { + return err + } + + t := fasttemplate.New(sqlStmt.String(), openVar, closeVar) + sqlStmt.Reset() + + _, err = t.Execute(&sqlStmt, varMap(ctx, req.Vars)) + + if err == errNoUserID && + authFailBlock == authFailBlockPerQuery && + authCheck(ctx) == false { + return errUnauthorized + } + + if err != nil { + return err + } + + finalSQL = sqlStmt.String() + + } else if err != nil { return err + + } else { + finalSQL = string(entry) } - var sqlStmt strings.Builder - - if err := pcompile.Compile(&sqlStmt, qc); err != nil { - return err - } - - t := fasttemplate.New(sqlStmt.String(), openVar, closeVar) - sqlStmt.Reset() - - _, err = t.Execute(&sqlStmt, varMap(ctx, req.Vars)) - - if err == errNoUserID && - authFailBlock == authFailBlockPerQuery && - authCheck(ctx) == false { - return errUnauthorized - } - - if err != nil { - return err - } - - finalSQL := sqlStmt.String() if conf.DebugLevel > 0 { fmt.Println(finalSQL) } @@ -55,6 +85,12 @@ func handleReq(ctx context.Context, w io.Writer, req *gqlReq) error { et := time.Now() resp := gqlResp{Data: json.RawMessage(root)} + if cacheEnabled { + if err = cache.Set(key, []byte(finalSQL)); err != nil { + return err + } + } + if conf.EnableTracing { resp.Extensions = &extensions{newTrace(st, et, qc)} }