diff --git a/go.mod b/go.mod index 78caf05..405b913 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( go.uber.org/zap v1.14.1 golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 golang.org/x/sys v0.0.0-20200413165638-669c56c373c4 // indirect - golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 + golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/ini.v1 v1.55.0 // indirect ) diff --git a/internal/serv/cmd_seed.go b/internal/serv/cmd_seed.go index 3f530b3..9fac3ff 100644 --- a/internal/serv/cmd_seed.go +++ b/internal/serv/cmd_seed.go @@ -18,6 +18,7 @@ import ( "github.com/dosco/super-graph/core" "github.com/gosimple/slug" "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/stdlib" "github.com/spf13/cobra" ) @@ -200,12 +201,12 @@ func importCSV(table, filename string) int64 { cols = append(cols, c.(string)) } - conn, err := acquireConn(db) + conn, err := stdlib.AcquireConn(db) if err != nil { log.Fatalf("ERR %v", err) } //nolint: errcheck - defer releaseConn(db, conn) + defer stdlib.ReleaseConn(db, conn) n, err := conn.CopyFrom( context.Background(), diff --git a/internal/serv/stdlib.go b/internal/serv/stdlib.go deleted file mode 100644 index 293113b..0000000 --- a/internal/serv/stdlib.go +++ /dev/null @@ -1,67 +0,0 @@ -package serv - -import ( - "context" - "database/sql" - "sync" - "time" - - errors "golang.org/x/xerrors" - - "github.com/jackc/pgx/v4" -) - -type ctxKey int - -var ctxKeyFakeTx ctxKey = 0 - -var errNotPgx = errors.New("not pgx *sql.DB") - -var ( - fakeTxMutex sync.Mutex - fakeTxConns map[*pgx.Conn]*sql.Tx -) - -func acquireConn(db *sql.DB) (*pgx.Conn, error) { - var conn *pgx.Conn - ctx := context.WithValue(context.Background(), ctxKeyFakeTx, &conn) - tx, err := db.BeginTx(ctx, nil) - if err != nil { - return nil, err - } - if conn == nil { - if err := tx.Rollback(); err != nil { - return nil, err - } - return nil, errNotPgx - } - - fakeTxMutex.Lock() - fakeTxConns[conn] = tx - fakeTxMutex.Unlock() - - return conn, nil -} - -func releaseConn(db *sql.DB, conn *pgx.Conn) error { - var tx *sql.Tx - var ok bool - - if conn.PgConn().IsBusy() || conn.PgConn().TxStatus() != 'I' { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - conn.Close(ctx) - } - - fakeTxMutex.Lock() - tx, ok = fakeTxConns[conn] - if ok { - delete(fakeTxConns, conn) - fakeTxMutex.Unlock() - } else { - fakeTxMutex.Unlock() - return errors.Errorf("can't release conn that is not acquired") - } - - return tx.Rollback() -} diff --git a/jsn/fuzz.go b/jsn/fuzz.go index 4b1e63e..23036ae 100644 --- a/jsn/fuzz.go +++ b/jsn/fuzz.go @@ -2,10 +2,55 @@ package jsn +import ( + "bytes" + "errors" +) + func Fuzz(data []byte) int { - if err := unifiedTest(data); err != nil { - return 0 + c := 0 + + if err := Validate(string(data)); err == nil { + c = 1 } - return 1 + if err := fuzzTest(data); err == nil { + c = 1 + } + + return c +} + +func fuzzTest(data []byte) error { + err1 := Validate(string(data)) + + var b1 bytes.Buffer + err2 := Filter(&b1, data, []string{"id", "full_name", "embed"}) + + path1 := [][]byte{[]byte("data"), []byte("users")} + Strip(data, path1) + + from := []Field{ + {[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)}, + {[]byte("__twitter_id"), []byte(`"ABC123"`)}, + } + + to := []Field{ + {[]byte("__twitter_id"), []byte(`"1234567890"`)}, + {[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)}, + } + + var b2 bytes.Buffer + err3 := Replace(&b2, data, from, to) + + Keys(data) + + var b3 bytes.Buffer + err4 := Clear(&b3, data) + + if err1 != nil || err2 != nil || err3 != nil || err4 != nil { + return errors.New("there was an error") + } + + return nil } diff --git a/jsn/fuzz_test.go b/jsn/fuzz_test.go index bbf493b..d99a24b 100644 --- a/jsn/fuzz_test.go +++ b/jsn/fuzz_test.go @@ -1,7 +1,11 @@ -package jsn +// +build gofuzz + +package jsn_test import ( "testing" + + "github.com/dosco/super-graph/jsn" ) func TestFuzzCrashers(t *testing.T) { @@ -55,6 +59,6 @@ func TestFuzzCrashers(t *testing.T) { } for _, f := range crashers { - _ = unifiedTest([]byte(f)) + _ = jsn.Fuzz([]byte(f)) } } diff --git a/jsn/json_test.go b/jsn/json_test.go index c473205..157fb98 100644 --- a/jsn/json_test.go +++ b/jsn/json_test.go @@ -1,4 +1,4 @@ -package jsn +package jsn_test import ( "bytes" @@ -6,6 +6,8 @@ import ( "io/ioutil" "strings" "testing" + + "github.com/dosco/super-graph/jsn" ) var ( @@ -171,13 +173,13 @@ var ( ) func TestGet(t *testing.T) { - values := Get([]byte(input1), [][]byte{ + values := jsn.Get([]byte(input1), [][]byte{ []byte("test_1a"), []byte("__twitter_id"), []byte("work_email"), }) - expected := []Field{ + expected := []jsn.Field{ {[]byte("test_1a"), []byte(`{ "__twitter_id": "ABCD" }`)}, {[]byte("__twitter_id"), []byte(`"ABCD"`)}, {[]byte("__twitter_id"), []byte(`"2048666903444506956"`)}, @@ -214,11 +216,11 @@ func TestGet(t *testing.T) { } func TestGet1(t *testing.T) { - values := Get([]byte(input5), [][]byte{ + values := jsn.Get([]byte(input5), [][]byte{ []byte("thread_slug"), }) - expected := []Field{ + expected := []jsn.Field{ {[]byte("thread_slug"), []byte(`"in-september-2018-slovak-police-stated-that-kuciak-7929"`)}, } @@ -238,11 +240,11 @@ func TestGet1(t *testing.T) { } func TestGet2(t *testing.T) { - values := Get([]byte(input6), [][]byte{ + values := jsn.Get([]byte(input6), [][]byte{ []byte("users_cursor"), []byte("threads_cursor"), }) - expected := []Field{ + expected := []jsn.Field{ {[]byte("threads_cursor"), []byte(`null`)}, {[]byte("threads_cursor"), []byte(`25`)}, {[]byte("users_cursor"), []byte(`3`)}, @@ -264,7 +266,7 @@ func TestGet2(t *testing.T) { } func TestGet3(t *testing.T) { - values := Get(input7, [][]byte{[]byte("data")}) + values := jsn.Get(input7, [][]byte{[]byte("data")}) v := values[0].Value if !bytes.Equal(v[len(v)-11:], []byte(`Rangnekar"}`)) { @@ -277,7 +279,7 @@ func TestGet4(t *testing.T) { exp = strings.ReplaceAll(exp, "@", "`") - values := Get(input8, [][]byte{[]byte("body")}) + values := jsn.Get(input8, [][]byte{[]byte("body")}) if string(values[0].Key) != "body" { t.Fatal("unexpected key") @@ -291,29 +293,29 @@ func TestGet4(t *testing.T) { func TestValue(t *testing.T) { v1 := []byte("12345") - if !bytes.Equal(Value(v1), v1) { + if !bytes.Equal(jsn.Value(v1), v1) { t.Fatal("Number value invalid") } v2 := []byte(`"12345"`) - if !bytes.Equal(Value(v2), []byte(`12345`)) { + if !bytes.Equal(jsn.Value(v2), []byte(`12345`)) { t.Fatal("String value invalid") } v3 := []byte(`{ "hello": "world" }`) - if Value(v3) != nil { - t.Fatal("Object value is not nil", Value(v3)) + if jsn.Value(v3) != nil { + t.Fatal("Object value is not nil", jsn.Value(v3)) } v4 := []byte(`[ "hello", "world" ]`) - if Value(v4) != nil { + if jsn.Value(v4) != nil { t.Fatal("List value is not nil") } } func TestFilter1(t *testing.T) { var b bytes.Buffer - err := Filter(&b, []byte(input2), []string{"id", "full_name", "embed"}) + err := jsn.Filter(&b, []byte(input2), []string{"id", "full_name", "embed"}) if err != nil { t.Error(err) } @@ -329,7 +331,7 @@ func TestFilter2(t *testing.T) { value := `[{"id":1,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":100,"amount_refunded":0,"date":"01/01/2019","application":null,"billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}, {"id":2,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":0,"date":"02/18/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}},{"id":3,"customer_id":"cus_2TbMGf3cl0","object":"charge","amount":150,"amount_refunded":50,"date":"03/21/2019","billing_details":{"address":"1 Infinity Drive","zipcode":"94024"}}]` var b bytes.Buffer - err := Filter(&b, []byte(value), []string{"id"}) + err := jsn.Filter(&b, []byte(value), []string{"id"}) if err != nil { t.Error(err) } @@ -343,7 +345,7 @@ func TestFilter2(t *testing.T) { func TestStrip(t *testing.T) { path1 := [][]byte{[]byte("data"), []byte("users")} - value1 := Strip([]byte(input3), path1) + value1 := jsn.Strip([]byte(input3), path1) expected := []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`) @@ -353,7 +355,7 @@ func TestStrip(t *testing.T) { } path2 := [][]byte{[]byte("boo"), []byte("hoo")} - value2 := Strip([]byte(input3), path2) + value2 := jsn.Strip([]byte(input3), path2) if !bytes.Equal(value2, []byte(input3)) { t.Log(value2) @@ -364,7 +366,7 @@ func TestStrip(t *testing.T) { func TestValidateTrue(t *testing.T) { json := []byte(` [{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`) - err := Validate(string(json)) + err := jsn.Validate(string(json)) if err != nil { t.Error(err) } @@ -373,7 +375,7 @@ func TestValidateTrue(t *testing.T) { func TestValidateFalse(t *testing.T) { json := []byte(` [{ "hello": 123"}]`) - err := Validate(string(json)) + err := jsn.Validate(string(json)) if err == nil { t.Error("JSON validation failed to detect invalid json") } @@ -382,12 +384,12 @@ func TestValidateFalse(t *testing.T) { func TestReplace(t *testing.T) { var buf bytes.Buffer - from := []Field{ + from := []jsn.Field{ {[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)}, {[]byte("__twitter_id"), []byte(`"ABC123"`)}, } - to := []Field{ + to := []jsn.Field{ {[]byte("__twitter_id"), []byte(`"1234567890"`)}, {[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)}, } @@ -412,7 +414,7 @@ func TestReplace(t *testing.T) { "__twitter_id":"1234567890" }] }` - err := Replace(&buf, []byte(input4), from, to) + err := jsn.Replace(&buf, []byte(input4), from, to) if err != nil { t.Fatal(err) } @@ -428,7 +430,7 @@ func TestReplaceEmpty(t *testing.T) { json := `{ "users" : [{"id":1,"full_name":"Sidney St[1]roman","email":"user0@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":2,"full_name":"Jerry Dickinson","email":"user1@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":3,"full_name":"Kenna Cassin","email":"user2@demo.com","__users_twitter_id":"2048666903444506956"}, {"id":4,"full_name":"Mr. Pat Parisian","email":"rodney@kautzer.biz","__users_twitter_id":"2048666903444506956"}, {"id":5,"full_name":"Bette Ebert","email":"janeenrath@goyette.com","__users_twitter_id":"2048666903444506956"}, {"id":6,"full_name":"Everett Kiehn","email":"michael@bartoletti.com","__users_twitter_id":"2048666903444506956"}, {"id":7,"full_name":"Katrina Cronin","email":"loretaklocko@framivolkman.org","__users_twitter_id":"2048666903444506956"}, {"id":8,"full_name":"Caroll Orn Sr.","email":"joannarau@hegmann.io","__users_twitter_id":"2048666903444506956"}, {"id":9,"full_name":"Gwendolyn Ziemann","email":"renaytoy@rutherford.co","__users_twitter_id":"2048666903444506956"}, {"id":10,"full_name":"Mrs. Rosann Fritsch","email":"holliemosciski@thiel.org","__users_twitter_id":"2048666903444506956"}, {"id":11,"full_name":"Arden Koss","email":"cristobalankunding@howewelch.org","__users_twitter_id":"2048666903444506956"}, {"id":12,"full_name":"Brenton Bauch PhD","email":"renee@miller.co","__users_twitter_id":"2048666903444506956"}, {"id":13,"full_name":"Daine Gleichner","email":"andrea@nienow.co","__users_twitter_id":"2048666903444506956"}] }` - err := Replace(&buf, []byte(json), []Field{}, []Field{}) + err := jsn.Replace(&buf, []byte(json), []jsn.Field{}, []jsn.Field{}) if err != nil { t.Fatal(err) } @@ -442,7 +444,7 @@ func TestReplaceEmpty(t *testing.T) { func TestKeys1(t *testing.T) { json := `[{"id":1,"posts": [{"title":"PT1-1","description":"PD1-1"}, {"title":"PT1-2","description":"PD1-2"}], "full_name":"FN1","email":"E1","books": [{"name":"BN1-1","description":"BD1-1"},{"name":"BN1-2","description":"BD1-2"},{"name":"BN1-2","description":"BD1-2"}]},{"id":1,"posts": [{"title":"PT1-1","description":"PD1-1"}, {"title":"PT1-2","description":"PD1-2"}], "full_name":"FN1","email":"E1","books": [{"name":"BN1-1","description":"BD1-1"},{"name":"BN1-2","description":"BD1-2"},{"name":"BN1-2","description":"BD1-2"}]},{"id":1,"posts": [{"title":"PT1-1","description":"PD1-1"}, {"title":"PT1-2","description":"PD1-2"}], "full_name":"FN1","email":"E1","books": [{"name":"BN1-1","description":"BD1-1"},{"name":"BN1-2","description":"BD1-2"},{"name":"BN1-2","description":"BD1-2"}]}]` - fields := Keys([]byte(json)) + fields := jsn.Keys([]byte(json)) exp := []string{ "id", "posts", "title", "description", "full_name", "email", "books", "name", "description", @@ -462,7 +464,7 @@ func TestKeys1(t *testing.T) { func TestKeys2(t *testing.T) { json := `{"id":1,"posts": [{"title":"PT1-1","description":"PD1-1"}, {"title":"PT1-2","description":"PD1-2"}], "full_name":"FN1","email":"E1","books": [{"name":"BN1-1","description":"BD1-1"},{"name":"BN1-2","description":"BD1-2"},{"name":"BN1-2","description":"BD1-2"}]}` - fields := Keys([]byte(json)) + fields := jsn.Keys([]byte(json)) exp := []string{ "id", "posts", "title", "description", "full_name", "email", "books", "name", "description", @@ -491,7 +493,7 @@ func TestKeys3(t *testing.T) { "user": 123 }` - fields := Keys([]byte(json)) + fields := jsn.Keys([]byte(json)) exp := []string{ "insert", "created_at", "test_1a", "type1", "type2", "name", "updated_at", "description", @@ -526,7 +528,7 @@ func TestClear(t *testing.T) { expected := `{"insert":{"created_at":"","test_1a":{"type1":"","type2":[{"a":0.0}]},"name":"","updated_at":"","description":""},"user":0.0,"tags":[]}` - err := Clear(&buf, []byte(json)) + err := jsn.Clear(&buf, []byte(json)) if err != nil { t.Fatal(err) } @@ -541,7 +543,7 @@ func BenchmarkGet(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { - Get([]byte(input1), [][]byte{[]byte("__twitter_id")}) + jsn.Get([]byte(input1), [][]byte{[]byte("__twitter_id")}) } } @@ -553,7 +555,7 @@ func BenchmarkFilter(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { - err := Filter(&buf, []byte(input2), keys) + err := jsn.Filter(&buf, []byte(input2), keys) if err != nil { b.Fatal(err) } @@ -566,19 +568,19 @@ func BenchmarkStrip(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { - Strip([]byte(input3), path) + jsn.Strip([]byte(input3), path) } } func BenchmarkReplace(b *testing.B) { var buf bytes.Buffer - from := []Field{ + from := []jsn.Field{ {[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)}, {[]byte("__twitter_id"), []byte(`"ABC123"`)}, } - to := []Field{ + to := []jsn.Field{ {[]byte("__twitter_id"), []byte(`"1234567890"`)}, {[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)}, } @@ -587,7 +589,7 @@ func BenchmarkReplace(b *testing.B) { b.ReportAllocs() for n := 0; n < b.N; n++ { - err := Replace(&buf, []byte(input4), from, to) + err := jsn.Replace(&buf, []byte(input4), from, to) if err != nil { b.Fatal(err) } diff --git a/jsn/test.go b/jsn/test.go deleted file mode 100644 index 3096ee2..0000000 --- a/jsn/test.go +++ /dev/null @@ -1,37 +0,0 @@ -package jsn - -import ( - "bytes" - "errors" -) - -func unifiedTest(data []byte) error { - err1 := Validate(string(data)) - - var b1 bytes.Buffer - err2 := Filter(&b1, data, []string{"id", "full_name", "embed"}) - - path1 := [][]byte{[]byte("data"), []byte("users")} - Strip(data, path1) - - from := []Field{ - {[]byte("__twitter_id"), []byte(`[{ "name": "hello" }, { "name": "world"}]`)}, - {[]byte("__twitter_id"), []byte(`"ABC123"`)}, - } - - to := []Field{ - {[]byte("__twitter_id"), []byte(`"1234567890"`)}, - {[]byte("some_list"), []byte(`[{"id":1,"embed":{"id":8}},{"id":2},{"id":3},{"id":4},{"id":5},{"id":6},{"id":7},{"id":8},{"id":9},{"id":10},{"id":11},{"id":12},{"id":13}]`)}, - } - - var b2 bytes.Buffer - err3 := Replace(&b2, data, from, to) - - Keys(data) - - if err1 != nil || err2 != nil || err3 != nil { - return errors.New("there was an error") - } - - return nil -}