diff --git a/.gitignore b/.gitignore index d4158dc..a286ab1 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ suppressions release .gofuzz *-fuzz.zip + diff --git a/Makefile b/Makefile index 3ed659f..fcc72d3 100644 --- a/Makefile +++ b/Makefile @@ -77,11 +77,10 @@ clean: run: clean @go run $(BUILD_FLAGS) main.go $(ARGS) -install: - @echo $(GOPATH) +install: build + @mv $(BINARY) $(GOPATH)/bin/$(BINARY) @echo "Commit Hash: `git rev-parse HEAD`" @echo "Old Hash: `shasum $(GOPATH)/bin/$(BINARY) 2>/dev/null | cut -c -32`" - @go install $(BUILD_FLAGS) cmd @echo "New Hash:" `shasum $(GOPATH)/bin/$(BINARY) 2>/dev/null | cut -c -32` uninstall: clean diff --git a/cmd/internal/serv/utils.go b/cmd/internal/serv/utils.go index 4a14e5e..2a88260 100644 --- a/cmd/internal/serv/utils.go +++ b/cmd/internal/serv/utils.go @@ -10,20 +10,9 @@ import ( "strings" "sync" - "github.com/cespare/xxhash/v2" "github.com/dosco/super-graph/jsn" ) -// nolint: errcheck -func mkkey(h *xxhash.Digest, k1 string, k2 string) uint64 { - h.WriteString(k1) - h.WriteString(k2) - v := h.Sum64() - h.Reset() - - return v -} - // nolint: errcheck func gqlHash(b string, vars []byte, role string) string { b = strings.TrimSpace(b) diff --git a/core/api.go b/core/api.go index 171c689..8f53db7 100644 --- a/core/api.go +++ b/core/api.go @@ -87,6 +87,7 @@ type SuperGraph struct { prepared map[string]*preparedItem roles map[string]*Role getRole *sql.Stmt + rmap map[uint64]*resolvFn abacEnabled bool anonExists bool qc *qcode.Compiler @@ -118,6 +119,10 @@ func NewSuperGraph(conf *Config, db *sql.DB) (*SuperGraph, error) { return nil, err } + if err := sg.initResolvers(); err != nil { + return nil, err + } + if len(conf.SecretKey) != 0 { sk := sha256.Sum256([]byte(conf.SecretKey)) conf.SecretKey = "" diff --git a/core/core.go b/core/core.go index 37688db..9ab0e04 100644 --- a/core/core.go +++ b/core/core.go @@ -89,25 +89,28 @@ func (sg *SuperGraph) initCompilers() error { func (c *scontext) execQuery() ([]byte, error) { var data []byte - // var st *stmt + var st *stmt var err error if c.sg.conf.UseAllowList { - data, _, err = c.resolvePreparedSQL() + data, st, err = c.resolvePreparedSQL() if err != nil { return nil, err } } else { - data, _, err = c.resolveSQL() + data, st, err = c.resolveSQL() if err != nil { return nil, err } } - return data, nil + if len(data) == 0 || st.skipped == 0 { + return data, nil + } - //return execRemoteJoin(st, data, c.req.hdr) + // return c.sg.execRemoteJoin(st, data, c.req.hdr) + return c.sg.execRemoteJoin(st, data, nil) } func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) { diff --git a/core/remote.go b/core/remote.go index af28352..6dbbd0c 100644 --- a/core/remote.go +++ b/core/remote.go @@ -1,253 +1,249 @@ package core -// import ( -// "bytes" -// "errors" -// "fmt" -// "net/http" -// "sync" +import ( + "bytes" + "errors" + "fmt" + "net/http" + "sync" -// "github.com/cespare/xxhash/v2" -// "github.com/dosco/super-graph/jsn" -// "github.com/dosco/super-graph/core/internal/qcode" -// ) + "github.com/cespare/xxhash/v2" + "github.com/dosco/super-graph/core/internal/qcode" + "github.com/dosco/super-graph/jsn" +) -// func execRemoteJoin(st *stmt, data []byte, hdr http.Header) ([]byte, error) { -// var err error +func (sg *SuperGraph) execRemoteJoin(st *stmt, data []byte, hdr http.Header) ([]byte, error) { + var err error -// if len(data) == 0 || st.skipped == 0 { -// return data, nil -// } + sel := st.qc.Selects + h := xxhash.New() -// sel := st.qc.Selects -// h := xxhash.New() + // fetch the field name used within the db response json + // that are used to mark insertion points and the mapping between + // those field names and their select objects + fids, sfmap := sg.parentFieldIds(h, sel, st.skipped) -// // fetch the field name used within the db response json -// // that are used to mark insertion points and the mapping between -// // those field names and their select objects -// fids, sfmap := parentFieldIds(h, sel, st.skipped) + // fetch the field values of the marked insertion points + // these values contain the id to be used with fetching remote data + from := jsn.Get(data, fids) + var to []jsn.Field -// // fetch the field values of the marked insertion points -// // these values contain the id to be used with fetching remote data -// from := jsn.Get(data, fids) -// var to []jsn.Field + switch { + case len(from) == 1: + to, err = sg.resolveRemote(hdr, h, from[0], sel, sfmap) -// switch { -// case len(from) == 1: -// to, err = resolveRemote(hdr, h, from[0], sel, sfmap) + case len(from) > 1: + to, err = sg.resolveRemotes(hdr, h, from, sel, sfmap) -// case len(from) > 1: -// to, err = resolveRemotes(hdr, h, from, sel, sfmap) + default: + return nil, errors.New("something wrong no remote ids found in db response") + } -// default: -// return nil, errors.New("something wrong no remote ids found in db response") -// } + if err != nil { + return nil, err + } -// if err != nil { -// return nil, err -// } + var ob bytes.Buffer -// var ob bytes.Buffer + err = jsn.Replace(&ob, data, from, to) + if err != nil { + return nil, err + } -// err = jsn.Replace(&ob, data, from, to) -// if err != nil { -// return nil, err -// } + return ob.Bytes(), nil +} -// return ob.Bytes(), nil -// } +func (sg *SuperGraph) resolveRemote( + hdr http.Header, + h *xxhash.Digest, + field jsn.Field, + sel []qcode.Select, + sfmap map[uint64]*qcode.Select) ([]jsn.Field, error) { -// func resolveRemote( -// hdr http.Header, -// h *xxhash.Digest, -// field jsn.Field, -// sel []qcode.Select, -// sfmap map[uint64]*qcode.Select) ([]jsn.Field, error) { + // replacement data for the marked insertion points + // key and value will be replaced by whats below + toA := [1]jsn.Field{} + to := toA[:1] -// // replacement data for the marked insertion points -// // key and value will be replaced by whats below -// toA := [1]jsn.Field{} -// to := toA[:1] + // use the json key to find the related Select object + k1 := xxhash.Sum64(field.Key) -// // use the json key to find the related Select object -// k1 := xxhash.Sum64(field.Key) + s, ok := sfmap[k1] + if !ok { + return nil, nil + } + p := sel[s.ParentID] -// s, ok := sfmap[k1] -// if !ok { -// return nil, nil -// } -// p := sel[s.ParentID] + // then use the Table nme in the Select and it's parent + // to find the resolver to use for this relationship + k2 := mkkey(h, s.Name, p.Name) -// // then use the Table nme in the Select and it's parent -// // to find the resolver to use for this relationship -// k2 := mkkey(h, s.Name, p.Name) + r, ok := sg.rmap[k2] + if !ok { + return nil, nil + } -// r, ok := rmap[k2] -// if !ok { -// return nil, nil -// } + id := jsn.Value(field.Value) + if len(id) == 0 { + return nil, nil + } -// id := jsn.Value(field.Value) -// if len(id) == 0 { -// return nil, nil -// } + //st := time.Now() -// //st := time.Now() + b, err := r.Fn(hdr, id) + if err != nil { + return nil, err + } -// b, err := r.Fn(hdr, id) -// if err != nil { -// return nil, err -// } + if len(r.Path) != 0 { + b = jsn.Strip(b, r.Path) + } -// if len(r.Path) != 0 { -// b = jsn.Strip(b, r.Path) -// } + var ob bytes.Buffer -// var ob bytes.Buffer + if len(s.Cols) != 0 { + err = jsn.Filter(&ob, b, colsToList(s.Cols)) + if err != nil { + return nil, err + } -// if len(s.Cols) != 0 { -// err = jsn.Filter(&ob, b, colsToList(s.Cols)) -// if err != nil { -// return nil, err -// } + } else { + ob.WriteString("null") + } -// } else { -// ob.WriteString("null") -// } + to[0] = jsn.Field{Key: []byte(s.FieldName), Value: ob.Bytes()} + return to, nil +} -// to[0] = jsn.Field{Key: []byte(s.FieldName), Value: ob.Bytes()} -// return to, nil -// } +func (sg *SuperGraph) resolveRemotes( + hdr http.Header, + h *xxhash.Digest, + from []jsn.Field, + sel []qcode.Select, + sfmap map[uint64]*qcode.Select) ([]jsn.Field, error) { -// func resolveRemotes( -// hdr http.Header, -// h *xxhash.Digest, -// from []jsn.Field, -// sel []qcode.Select, -// sfmap map[uint64]*qcode.Select) ([]jsn.Field, error) { + // replacement data for the marked insertion points + // key and value will be replaced by whats below + to := make([]jsn.Field, len(from)) -// // replacement data for the marked insertion points -// // key and value will be replaced by whats below -// to := make([]jsn.Field, len(from)) + var wg sync.WaitGroup + wg.Add(len(from)) -// var wg sync.WaitGroup -// wg.Add(len(from)) + var cerr error -// var cerr error + for i, id := range from { -// for i, id := range from { + // use the json key to find the related Select object + k1 := xxhash.Sum64(id.Key) -// // use the json key to find the related Select object -// k1 := xxhash.Sum64(id.Key) + s, ok := sfmap[k1] + if !ok { + return nil, nil + } + p := sel[s.ParentID] -// s, ok := sfmap[k1] -// if !ok { -// return nil, nil -// } -// p := sel[s.ParentID] + // then use the Table nme in the Select and it's parent + // to find the resolver to use for this relationship + k2 := mkkey(h, s.Name, p.Name) -// // then use the Table nme in the Select and it's parent -// // to find the resolver to use for this relationship -// k2 := mkkey(h, s.Name, p.Name) + r, ok := sg.rmap[k2] + if !ok { + return nil, nil + } -// r, ok := rmap[k2] -// if !ok { -// return nil, nil -// } + id := jsn.Value(id.Value) + if len(id) == 0 { + return nil, nil + } -// id := jsn.Value(id.Value) -// if len(id) == 0 { -// return nil, nil -// } + go func(n int, id []byte, s *qcode.Select) { + defer wg.Done() -// go func(n int, id []byte, s *qcode.Select) { -// defer wg.Done() + //st := time.Now() -// //st := time.Now() + b, err := r.Fn(hdr, id) + if err != nil { + cerr = fmt.Errorf("%s: %s", s.Name, err) + return + } -// b, err := r.Fn(hdr, id) -// if err != nil { -// cerr = fmt.Errorf("%s: %s", s.Name, err) -// return -// } + if len(r.Path) != 0 { + b = jsn.Strip(b, r.Path) + } -// if len(r.Path) != 0 { -// b = jsn.Strip(b, r.Path) -// } + var ob bytes.Buffer -// var ob bytes.Buffer + if len(s.Cols) != 0 { + err = jsn.Filter(&ob, b, colsToList(s.Cols)) + if err != nil { + cerr = fmt.Errorf("%s: %s", s.Name, err) + return + } -// if len(s.Cols) != 0 { -// err = jsn.Filter(&ob, b, colsToList(s.Cols)) -// if err != nil { -// cerr = fmt.Errorf("%s: %s", s.Name, err) -// return -// } + } else { + ob.WriteString("null") + } -// } else { -// ob.WriteString("null") -// } + to[n] = jsn.Field{Key: []byte(s.FieldName), Value: ob.Bytes()} + }(i, id, s) + } + wg.Wait() -// to[n] = jsn.Field{Key: []byte(s.FieldName), Value: ob.Bytes()} -// }(i, id, s) -// } -// wg.Wait() + return to, cerr +} -// return to, cerr -// } +func (sg *SuperGraph) parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) ( + [][]byte, + map[uint64]*qcode.Select) { -// func parentFieldIds(h *xxhash.Digest, sel []qcode.Select, skipped uint32) ( -// [][]byte, -// map[uint64]*qcode.Select) { + c := 0 + for i := range sel { + s := &sel[i] + if isSkipped(skipped, uint32(s.ID)) { + c++ + } + } -// c := 0 -// for i := range sel { -// s := &sel[i] -// if isSkipped(skipped, uint32(s.ID)) { -// c++ -// } -// } + // list of keys (and it's related value) to extract from + // the db json response + fm := make([][]byte, c) -// // list of keys (and it's related value) to extract from -// // the db json response -// fm := make([][]byte, c) + // mapping between the above extracted key and a Select + // object + sm := make(map[uint64]*qcode.Select, c) + n := 0 -// // mapping between the above extracted key and a Select -// // object -// sm := make(map[uint64]*qcode.Select, c) -// n := 0 + for i := range sel { + s := &sel[i] -// for i := range sel { -// s := &sel[i] + if !isSkipped(skipped, uint32(s.ID)) { + continue + } -// if !isSkipped(skipped, uint32(s.ID)) { -// continue -// } + p := sel[s.ParentID] + k := mkkey(h, s.Name, p.Name) -// p := sel[s.ParentID] -// k := mkkey(h, s.Name, p.Name) + if r, ok := sg.rmap[k]; ok { + fm[n] = r.IDField + n++ -// if r, ok := rmap[k]; ok { -// fm[n] = r.IDField -// n++ + k := xxhash.Sum64(r.IDField) + sm[k] = s + } + } -// k := xxhash.Sum64(r.IDField) -// sm[k] = s -// } -// } + return fm, sm +} -// return fm, sm -// } +func isSkipped(n uint32, pos uint32) bool { + return ((n & (1 << pos)) != 0) +} -// func isSkipped(n uint32, pos uint32) bool { -// return ((n & (1 << pos)) != 0) -// } +func colsToList(cols []qcode.Column) []string { + var f []string -// func colsToList(cols []qcode.Column) []string { -// var f []string - -// for i := range cols { -// f = append(f, cols[i].Name) -// } -// return f -// } + for i := range cols { + f = append(f, cols[i].Name) + } + return f +} diff --git a/core/resolve.go b/core/resolve.go index 5379fe4..8987103 100644 --- a/core/resolve.go +++ b/core/resolve.go @@ -6,90 +6,90 @@ import ( "net/http" "strings" + "github.com/cespare/xxhash/v2" + "github.com/dosco/super-graph/core/internal/psql" "github.com/dosco/super-graph/jsn" ) -var ( - rmap map[uint64]*resolvFn -) - type resolvFn struct { IDField []byte Path [][]byte Fn func(h http.Header, id []byte) ([]byte, error) } -// func initResolvers() { -// var err error -// rmap = make(map[uint64]*resolvFn) +func (sg *SuperGraph) initResolvers() error { + var err error + sg.rmap = make(map[uint64]*resolvFn) -// for _, t := range conf.Tables { -// err = initRemotes(t) -// if err != nil { -// break -// } -// } + for _, t := range sg.conf.Tables { + err = sg.initRemotes(t) + if err != nil { + break + } + } -// if err != nil { -// errlog.Fatal().Err(err).Msg("failed to initialize resolvers") -// } -// } + if err != nil { + return fmt.Errorf("failed to initialize resolvers: %v", err) + } -// func initRemotes(t Table) error { -// h := xxhash.New() + return nil +} -// for _, r := range t.Remotes { -// // defines the table column to be used as an id in the -// // remote request -// idcol := r.ID +func (sg *SuperGraph) initRemotes(t Table) error { + h := xxhash.New() -// // if no table column specified in the config then -// // use the primary key of the table as the id -// if len(idcol) == 0 { -// pcol, err := pcompile.IDColumn(t.Name) -// if err != nil { -// return err -// } -// idcol = pcol.Key -// } -// idk := fmt.Sprintf("__%s_%s", t.Name, idcol) + for _, r := range t.Remotes { + // defines the table column to be used as an id in the + // remote request + idcol := r.ID -// // register a relationship between the remote data -// // and the database table + // if no table column specified in the config then + // use the primary key of the table as the id + if len(idcol) == 0 { + pcol, err := sg.pc.IDColumn(t.Name) + if err != nil { + return err + } + idcol = pcol.Key + } + idk := fmt.Sprintf("__%s_%s", t.Name, idcol) -// val := &psql.DBRel{Type: psql.RelRemote} -// val.Left.Col = idcol -// val.Right.Col = idk + // register a relationship between the remote data + // and the database table -// err := pcompile.AddRelationship(strings.ToLower(r.Name), t.Name, val) -// if err != nil { -// return err -// } + val := &psql.DBRel{Type: psql.RelRemote} + val.Left.Col = idcol + val.Right.Col = idk -// // the function thats called to resolve this remote -// // data request -// fn := buildFn(r) + err := sg.pc.AddRelationship(sanitize(r.Name), t.Name, val) + if err != nil { + return err + } -// path := [][]byte{} -// for _, p := range strings.Split(r.Path, ".") { -// path = append(path, []byte(p)) -// } + // the function thats called to resolve this remote + // data request + fn := buildFn(r) -// rf := &resolvFn{ -// IDField: []byte(idk), -// Path: path, -// Fn: fn, -// } + path := [][]byte{} + for _, p := range strings.Split(r.Path, ".") { + path = append(path, []byte(p)) + } -// // index resolver obj by parent and child names -// rmap[mkkey(h, r.Name, t.Name)] = rf + rf := &resolvFn{ + IDField: []byte(idk), + Path: path, + Fn: fn, + } -// // index resolver obj by IDField -// rmap[xxhash.Sum64(rf.IDField)] = rf -// } + // index resolver obj by parent and child names + sg.rmap[mkkey(h, r.Name, t.Name)] = rf -// return nil -// } + // index resolver obj by IDField + sg.rmap[xxhash.Sum64(rf.IDField)] = rf + } + + return nil +} func buildFn(r Remote) func(http.Header, []byte) ([]byte, error) { reqURL := strings.Replace(r.URL, "$id", "%s", 1) @@ -114,12 +114,9 @@ func buildFn(r Remote) func(http.Header, []byte) ([]byte, error) { req.Header.Set(v, hdr.Get(v)) } - // logger.Debug().Str("uri", uri).Msg("Remote Join") - res, err := client.Do(req) if err != nil { - // errlog.Error().Err(err).Msgf("Failed to connect to: %s", uri) - return nil, err + return nil, fmt.Errorf("failed to connect to '%s': %v", uri, err) } defer res.Body.Close() diff --git a/core/utils.go b/core/utils.go new file mode 100644 index 0000000..c4e0147 --- /dev/null +++ b/core/utils.go @@ -0,0 +1,15 @@ +package core + +import ( + "github.com/cespare/xxhash/v2" +) + +// nolint: errcheck +func mkkey(h *xxhash.Digest, k1 string, k2 string) uint64 { + h.WriteString(k1) + h.WriteString(k2) + v := h.Sum64() + h.Reset() + + return v +} diff --git a/go.mod b/go.mod index 4ce29a2..349bc12 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/adjust/gorails v0.0.0-20171013043634-2786ed0c03d3 github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b github.com/brianvoe/gofakeit v3.18.0+incompatible + github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.1.0 github.com/daaku/go.zipexe v1.0.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible