wpetit/super-graph
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Remove config package

Browse Source
This commit is contained in:
Vikram Rangnekar
2020-04-11 02:45:06 -04:00
parent 7831d27345
commit a266517d17
51 changed files with 2856 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
core/api.go
View File

@ -9,7 +9,6 @@
"database/sql"
"fmt"
"time"
"github.com/dosco/super-graph/config"
"github.com/dosco/super-graph/core"
_ "github.com/jackc/pgx/v4/stdlib"
)
@ -20,7 +19,7 @@
log.Fatalf(err)
}
conf, err := config.NewConfig("./config")
conf, err := core.ReadInConfig("./config/dev.yml")
if err != nil {
log.Fatalf(err)
}
@ -53,10 +52,9 @@ import (
"crypto/sha256"
"database/sql"
"encoding/json"
"fmt"
"log"
_log "log"
"os"
"github.com/dosco/super-graph/config"
"github.com/dosco/super-graph/core/internal/allow"
"github.com/dosco/super-graph/core/internal/crypto"
"github.com/dosco/super-graph/core/internal/psql"
@ -80,36 +78,32 @@ const (
// SuperGraph struct is an instance of the Super Graph engine it holds all the required information like
// datase schemas, relationships, etc that the GraphQL to SQL compiler would need to do it's job.
type SuperGraph struct {
conf *config.Config
db *sql.DB
schema *psql.DBSchema
allowList *allow.List
encKey [32]byte
prepared map[string]*preparedItem
getRole *sql.Stmt
qc *qcode.Compiler
pc *psql.Compiler
}
// NewConfig functions initializes config using a config.Core struct
func NewConfig(core config.Core, configPath string, logger *log.Logger) (*config.Config, error) {
c, err := config.NewConfigFrom(&config.Config{Core: core}, configPath, logger)
if err != nil {
return nil, err
}
return c, nil
conf *Config
db *sql.DB
log *_log.Logger
schema *psql.DBSchema
allowList *allow.List
encKey [32]byte
prepared map[string]*preparedItem
roles map[string]*Role
getRole *sql.Stmt
abacEnabled bool
anonExists bool
qc *qcode.Compiler
pc *psql.Compiler
}
// NewSuperGraph creates the SuperGraph struct, this involves querying the database to learn its
// schemas and relationships
func NewSuperGraph(conf *config.Config, db *sql.DB) (*SuperGraph, error) {
if !conf.IsValid() {
return nil, fmt.Errorf("invalid config")
}
func NewSuperGraph(conf *Config, db *sql.DB) (*SuperGraph, error) {
sg := &SuperGraph{
conf: conf,
db: db,
log: _log.New(os.Stdout, "", 0),
}
if err := sg.initConfig(); err != nil {
return nil, err
}
if err := sg.initCompilers(); err != nil {

11
core/build.go
View File

@ -7,13 +7,12 @@ import (
"fmt"
"io"
"github.com/dosco/super-graph/config"
"github.com/dosco/super-graph/core/internal/psql"
"github.com/dosco/super-graph/core/internal/qcode"
)
type stmt struct {
role *config.Role
role *Role
qc *qcode.QCode
skipped uint32
sql string
@ -29,7 +28,7 @@ func (sg *SuperGraph) buildStmt(qt qcode.QType, query, vars []byte, role string)
return sg.buildRoleStmt(query, vars, "anon")
}
if sg.conf.IsABACEnabled() {
if sg.abacEnabled {
return sg.buildMultiStmt(query, vars)
}
@ -41,8 +40,8 @@ func (sg *SuperGraph) buildStmt(qt qcode.QType, query, vars []byte, role string)
}
func (sg *SuperGraph) buildRoleStmt(query, vars []byte, role string) ([]stmt, error) {
ro := sg.conf.GetRole(role)
if ro == nil {
ro, ok := sg.roles[role]
if !ok {
return nil, fmt.Errorf(`roles '%s' not defined in c.sg.config`, role)
}
@ -168,7 +167,7 @@ func (sg *SuperGraph) renderUserQuery(stmts []stmt) (string, error) {
return w.String(), nil
}
func (sg *SuperGraph) hasTablesWithConfig(qc *qcode.QCode, role *config.Role) bool {
func (sg *SuperGraph) hasTablesWithConfig(qc *qcode.QCode, role *Role) bool {
for _, id := range qc.Roots {
t, err := sg.schema.GetTable(qc.Selects[id].Name)
if err != nil {

286
core/config.go
View File

@ -2,164 +2,162 @@ package core
import (
"fmt"
"path"
"strings"
"github.com/dosco/super-graph/config"
"github.com/dosco/super-graph/core/internal/psql"
"github.com/dosco/super-graph/core/internal/qcode"
"github.com/spf13/viper"
)
func addTables(c *config.Config, di *psql.DBInfo) error {
for _, t := range c.Tables {
if len(t.Table) == 0 || len(t.Columns) == 0 {
continue
// Core struct contains core specific config value
type Config struct {
SecretKey string `mapstructure:"secret_key"`
UseAllowList bool `mapstructure:"use_allow_list"`
AllowListFile string `mapstructure:"allow_list_file"`
SetUserID bool `mapstructure:"set_user_id"`
Vars map[string]string `mapstructure:"variables"`
Blocklist []string
Tables []Table
RolesQuery string `mapstructure:"roles_query"`
Roles []Role
Inflections map[string]string
}
// Table struct defines a database table
type Table struct {
Name string
Table string
Blocklist []string
Remotes []Remote
Columns []Column
}
// Column struct defines a database column
type Column struct {
Name string
Type string
ForeignKey string `mapstructure:"related_to"`
}
// Remote struct defines a remote API endpoint
type Remote struct {
Name string
ID string
Path string
URL string
Debug bool
PassHeaders []string `mapstructure:"pass_headers"`
SetHeaders []struct {
Name string
Value string
} `mapstructure:"set_headers"`
}
// Role struct contains role specific access control values for for all database tables
type Role struct {
Name string
Match string
Tables []RoleTable
tm map[string]*RoleTable
}
// RoleTable struct contains role specific access control values for a database table
type RoleTable struct {
Name string
Query Query
Insert Insert
Update Update
Delete Delete
}
// Query struct contains access control values for query operations
type Query struct {
Limit int
Filters []string
Columns []string
DisableFunctions bool `mapstructure:"disable_functions"`
Block bool
}
// Insert struct contains access control values for insert operations
type Insert struct {
Filters []string
Columns []string
Presets map[string]string
Block bool
}
// Insert struct contains access control values for update operations
type Update struct {
Filters []string
Columns []string
Presets map[string]string
Block bool
}
// Delete struct contains access control values for delete operations
type Delete struct {
Filters []string
Columns []string
Block bool
}
// ReadInConfig function reads in the config file for the environment specified in the GO_ENV
// environment variable. This is the best way to create a new Super Graph config.
func ReadInConfig(configFile string) (*Config, error) {
cpath := path.Dir(configFile)
cfile := path.Base(configFile)
vi := newViper(cpath, cfile)
if err := vi.ReadInConfig(); err != nil {
return nil, err
}
inherits := vi.GetString("inherits")
if len(inherits) != 0 {
vi = newViper(cpath, inherits)
if err := vi.ReadInConfig(); err != nil {
return nil, err
}
if err := addTable(di, t.Columns, t); err != nil {
return err
if vi.IsSet("inherits") {
return nil, fmt.Errorf("inherited config (%s) cannot itself inherit (%s)",
inherits,
vi.GetString("inherits"))
}
}
return nil
}
func addTable(di *psql.DBInfo, cols []config.Column, t config.Table) error {
bc, ok := di.GetColumn(t.Table, t.Name)
if !ok {
return fmt.Errorf(
"Column '%s' not found on table '%s'",
t.Name, t.Table)
}
vi.SetConfigName(cfile)
if bc.Type != "json" && bc.Type != "jsonb" {
return fmt.Errorf(
"Column '%s' in table '%s' is of type '%s'. Only JSON or JSONB is valid",
t.Name, t.Table, bc.Type)
}
table := psql.DBTable{
Name: t.Name,
Key: strings.ToLower(t.Name),
Type: bc.Type,
}
columns := make([]psql.DBColumn, 0, len(cols))
for i := range cols {
c := cols[i]
columns = append(columns, psql.DBColumn{
Name: c.Name,
Key: strings.ToLower(c.Name),
Type: c.Type,
})
}
di.AddTable(table, columns)
bc.FKeyTable = t.Name
return nil
}
func addForeignKeys(c *config.Config, di *psql.DBInfo) error {
for _, t := range c.Tables {
for _, c := range t.Columns {
if len(c.ForeignKey) == 0 {
continue
}
if err := addForeignKey(di, c, t); err != nil {
return err
}
}
}
return nil
}
func addForeignKey(di *psql.DBInfo, c config.Column, t config.Table) error {
c1, ok := di.GetColumn(t.Name, c.Name)
if !ok {
return fmt.Errorf(
"Invalid table '%s' or column '%s' in config.Config",
t.Name, c.Name)
}
v := strings.SplitN(c.ForeignKey, ".", 2)
if len(v) != 2 {
return fmt.Errorf(
"Invalid foreign_key in config.Config for table '%s' and column '%s",
t.Name, c.Name)
}
fkt, fkc := v[0], v[1]
c2, ok := di.GetColumn(fkt, fkc)
if !ok {
return fmt.Errorf(
"Invalid foreign_key in config.Config for table '%s' and column '%s",
t.Name, c.Name)
}
c1.FKeyTable = fkt
c1.FKeyColID = []int16{c2.ID}
return nil
}
func addRoles(c *config.Config, qc *qcode.Compiler) error {
for _, r := range c.Roles {
for _, t := range r.Tables {
if err := addRole(qc, r, t); err != nil {
return err
}
if err := vi.MergeInConfig(); err != nil {
return nil, err
}
}
return nil
c := &Config{}
if err := vi.Unmarshal(&c); err != nil {
return nil, fmt.Errorf("failed to decode config, %v", err)
}
if len(c.AllowListFile) == 0 {
c.AllowListFile = path.Join(cpath, "allow.list")
}
return c, nil
}
func addRole(qc *qcode.Compiler, r config.Role, t config.RoleTable) error {
blockFilter := []string{"false"}
func newViper(configPath, configFile string) *viper.Viper {
vi := viper.New()
query := qcode.QueryConfig{
Limit: t.Query.Limit,
Filters: t.Query.Filters,
Columns: t.Query.Columns,
DisableFunctions: t.Query.DisableFunctions,
}
vi.SetEnvPrefix("SG")
vi.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
vi.AutomaticEnv()
if t.Query.Block {
query.Filters = blockFilter
}
vi.SetConfigName(configFile)
vi.AddConfigPath(configPath)
vi.AddConfigPath("./config")
insert := qcode.InsertConfig{
Filters: t.Insert.Filters,
Columns: t.Insert.Columns,
Presets: t.Insert.Presets,
}
if t.Insert.Block {
insert.Filters = blockFilter
}
update := qcode.UpdateConfig{
Filters: t.Update.Filters,
Columns: t.Update.Columns,
Presets: t.Update.Presets,
}
if t.Update.Block {
update.Filters = blockFilter
}
delete := qcode.DeleteConfig{
Filters: t.Delete.Filters,
Columns: t.Delete.Columns,
}
if t.Delete.Block {
delete.Filters = blockFilter
}
return qc.AddRole(r.Name, t.Name, qcode.TRConfig{
Query: query,
Insert: insert,
Update: update,
Delete: delete,
})
return vi
}

10
core/core.go
View File

@ -63,7 +63,7 @@ func (sg *SuperGraph) initCompilers() error {
return err
}
sg.schema, err = psql.NewDBSchema(di, sg.conf.GetDBTableAliases())
sg.schema, err = psql.NewDBSchema(di, getDBTableAliases(sg.conf))
if err != nil {
return err
}
@ -92,7 +92,7 @@ func (c *scontext) execQuery() ([]byte, error) {
// var st *stmt
var err error
if c.sg.conf.Production {
if c.sg.conf.UseAllowList {
data, _, err = c.resolvePreparedSQL()
if err != nil {
return nil, err
@ -115,7 +115,7 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
var err error
mutation := (c.res.op == qcode.QTMutation)
useRoleQuery := c.sg.conf.IsABACEnabled() && mutation
useRoleQuery := c.sg.abacEnabled && mutation
useTx := useRoleQuery || c.sg.conf.SetUserID
if useTx {
@ -148,7 +148,7 @@ func (c *scontext) resolvePreparedSQL() ([]byte, *stmt, error) {
c.res.role = role
ps, ok := prepared[stmtHash(c.res.name, role)]
ps, ok := c.sg.prepared[stmtHash(c.res.name, role)]
if !ok {
return nil, nil, errNotFound
}
@ -198,7 +198,7 @@ func (c *scontext) resolveSQL() ([]byte, *stmt, error) {
var err error
mutation := (c.res.op == qcode.QTMutation)
useRoleQuery := c.sg.conf.IsABACEnabled() && mutation
useRoleQuery := c.sg.abacEnabled && mutation
useTx := useRoleQuery || c.sg.conf.SetUserID
if useTx {

284
core/init.go Normal file
View File

@ -0,0 +1,284 @@
package core
import (
"fmt"
"regexp"
"strings"
"unicode"
"github.com/dosco/super-graph/core/internal/psql"
"github.com/dosco/super-graph/core/internal/qcode"
"github.com/gobuffalo/flect"
)
func (sg *SuperGraph) initConfig() error {
c := sg.conf
for k, v := range c.Inflections {
flect.AddPlural(k, v)
}
// Variables: Validate and sanitize
for k, v := range c.Vars {
c.Vars[k] = sanitizeVars(v)
}
// Tables: Validate and sanitize
tm := make(map[string]struct{})
for i := 0; i < len(c.Tables); i++ {
t := &c.Tables[i]
t.Name = flect.Pluralize(strings.ToLower(t.Name))
if _, ok := tm[t.Name]; ok {
sg.conf.Tables = append(c.Tables[:i], c.Tables[i+1:]...)
sg.log.Printf("WRN duplicate table found: %s", t.Name)
}
tm[t.Name] = struct{}{}
t.Table = flect.Pluralize(strings.ToLower(t.Table))
}
sg.roles = make(map[string]*Role)
for i := 0; i < len(c.Roles); i++ {
role := &c.Roles[i]
role.Name = sanitize(role.Name)
if _, ok := sg.roles[role.Name]; ok {
c.Roles = append(c.Roles[:i], c.Roles[i+1:]...)
sg.log.Printf("WRN duplicate role found: %s", role.Name)
}
role.Match = sanitize(role.Match)
role.tm = make(map[string]*RoleTable)
for n, table := range role.Tables {
role.tm[table.Name] = &role.Tables[n]
}
sg.roles[role.Name] = role
}
// If user role not defined then create it
if _, ok := sg.roles["user"]; !ok {
ur := Role{
Name: "user",
tm: make(map[string]*RoleTable),
}
c.Roles = append(c.Roles, ur)
sg.roles["user"] = &ur
}
// Roles: validate and sanitize
c.RolesQuery = sanitize(c.RolesQuery)
if len(c.RolesQuery) == 0 {
sg.log.Printf("WRN roles_query not defined: attribute based access control disabled")
}
_, userExists := sg.roles["user"]
_, sg.anonExists = sg.roles["anon"]
sg.abacEnabled = userExists && len(c.RolesQuery) != 0
return nil
}
func getDBTableAliases(c *Config) map[string][]string {
m := make(map[string][]string, len(c.Tables))
for i := range c.Tables {
t := c.Tables[i]
if len(t.Table) == 0 || len(t.Columns) != 0 {
continue
}
m[t.Table] = append(m[t.Table], t.Name)
}
return m
}
func addTables(c *Config, di *psql.DBInfo) error {
for _, t := range c.Tables {
if len(t.Table) == 0 || len(t.Columns) == 0 {
continue
}
if err := addTable(di, t.Columns, t); err != nil {
return err
}
}
return nil
}
func addTable(di *psql.DBInfo, cols []Column, t Table) error {
bc, ok := di.GetColumn(t.Table, t.Name)
if !ok {
return fmt.Errorf(
"Column '%s' not found on table '%s'",
t.Name, t.Table)
}
if bc.Type != "json" && bc.Type != "jsonb" {
return fmt.Errorf(
"Column '%s' in table '%s' is of type '%s'. Only JSON or JSONB is valid",
t.Name, t.Table, bc.Type)
}
table := psql.DBTable{
Name: t.Name,
Key: strings.ToLower(t.Name),
Type: bc.Type,
}
columns := make([]psql.DBColumn, 0, len(cols))
for i := range cols {
c := cols[i]
columns = append(columns, psql.DBColumn{
Name: c.Name,
Key: strings.ToLower(c.Name),
Type: c.Type,
})
}
di.AddTable(table, columns)
bc.FKeyTable = t.Name
return nil
}
func addForeignKeys(c *Config, di *psql.DBInfo) error {
for _, t := range c.Tables {
for _, c := range t.Columns {
if len(c.ForeignKey) == 0 {
continue
}
if err := addForeignKey(di, c, t); err != nil {
return err
}
}
}
return nil
}
func addForeignKey(di *psql.DBInfo, c Column, t Table) error {
c1, ok := di.GetColumn(t.Name, c.Name)
if !ok {
return fmt.Errorf(
"Invalid table '%s' or column '%s' in Config",
t.Name, c.Name)
}
v := strings.SplitN(c.ForeignKey, ".", 2)
if len(v) != 2 {
return fmt.Errorf(
"Invalid foreign_key in Config for table '%s' and column '%s",
t.Name, c.Name)
}
fkt, fkc := v[0], v[1]
c2, ok := di.GetColumn(fkt, fkc)
if !ok {
return fmt.Errorf(
"Invalid foreign_key in Config for table '%s' and column '%s",
t.Name, c.Name)
}
c1.FKeyTable = fkt
c1.FKeyColID = []int16{c2.ID}
return nil
}
func addRoles(c *Config, qc *qcode.Compiler) error {
for _, r := range c.Roles {
for _, t := range r.Tables {
if err := addRole(qc, r, t); err != nil {
return err
}
}
}
return nil
}
func addRole(qc *qcode.Compiler, r Role, t RoleTable) error {
blockFilter := []string{"false"}
query := qcode.QueryConfig{
Limit: t.Query.Limit,
Filters: t.Query.Filters,
Columns: t.Query.Columns,
DisableFunctions: t.Query.DisableFunctions,
}
if t.Query.Block {
query.Filters = blockFilter
}
insert := qcode.InsertConfig{
Filters: t.Insert.Filters,
Columns: t.Insert.Columns,
Presets: t.Insert.Presets,
}
if t.Insert.Block {
insert.Filters = blockFilter
}
update := qcode.UpdateConfig{
Filters: t.Update.Filters,
Columns: t.Update.Columns,
Presets: t.Update.Presets,
}
if t.Update.Block {
update.Filters = blockFilter
}
delete := qcode.DeleteConfig{
Filters: t.Delete.Filters,
Columns: t.Delete.Columns,
}
if t.Delete.Block {
delete.Filters = blockFilter
}
return qc.AddRole(r.Name, t.Name, qcode.TRConfig{
Query: query,
Insert: insert,
Update: update,
Delete: delete,
})
}
func (r *Role) GetTable(name string) *RoleTable {
return r.tm[name]
}
func sanitize(value string) string {
return strings.ToLower(strings.TrimSpace(value))
}
var (
varRe1 = regexp.MustCompile(`(?mi)\$([a-zA-Z0-9_.]+)`)
varRe2 = regexp.MustCompile(`\{\{([a-zA-Z0-9_.]+)\}\}`)
)
func sanitizeVars(s string) string {
s0 := varRe1.ReplaceAllString(s, `{{$1}}`)
s1 := strings.Map(func(r rune) rune {
if unicode.IsSpace(r) {
return ' '
}
return r
}, s0)
return varRe2.ReplaceAllStringFunc(s1, func(m string) string {
return strings.ToLower(m)
})
}

11
core/internal/allow/allow.go
View File

@ -7,7 +7,6 @@ import (
"fmt"
"io/ioutil"
"os"
"path"
"sort"
"strings"
)
@ -35,11 +34,11 @@ type Config struct {
Persist bool
}
func New(cpath string, conf Config) (*List, error) {
func New(filename string, conf Config) (*List, error) {
al := List{}
if len(cpath) != 0 {
fp := path.Join(cpath, "allow.list")
if len(filename) != 0 {
fp := filename
if _, err := os.Stat(fp); err == nil {
al.filepath = fp
@ -73,10 +72,10 @@ func New(cpath string, conf Config) (*List, error) {
return nil, errors.New("allow.list not found")
}
if len(cpath) == 0 {
if len(filename) == 0 {
al.filepath = "./config/allow.list"
} else {
al.filepath = path.Join(cpath, "allow.list")
al.filepath = filename
}
}

21
core/prepare.go
View File

@ -23,17 +23,13 @@ type preparedItem struct {
roleArg bool
}
var (
prepared map[string]*preparedItem
)
func (sg *SuperGraph) initPrepared() error {
ct := context.Background()
if sg.allowList.IsPersist() {
return nil
}
prepared = make(map[string]*preparedItem)
sg.prepared = make(map[string]*preparedItem)
tx, err := sg.db.BeginTx(ct, nil)
if err != nil {
@ -100,7 +96,7 @@ func (sg *SuperGraph) prepareStmt(item allow.Item) error {
var stmts1 []stmt
var err error
if sg.conf.IsABACEnabled() {
if sg.abacEnabled {
stmts1, err = sg.buildMultiStmt(qb, vars)
} else {
stmts1, err = sg.buildRoleStmt(qb, vars, "user")
@ -117,7 +113,7 @@ func (sg *SuperGraph) prepareStmt(item allow.Item) error {
return err
}
if sg.conf.IsAnonRoleDefined() {
if sg.anonExists {
// logger.Debug().Msgf("Prepared statement 'query %s' (anon)", item.Name)
stmts2, err := sg.buildRoleStmt(qb, vars, "anon")
@ -184,7 +180,7 @@ func (sg *SuperGraph) prepare(ct context.Context, tx *sql.Tx, st []stmt, key str
func (sg *SuperGraph) prepareRoleStmt(tx *sql.Tx) error {
var err error
if !sg.conf.IsABACEnabled() {
if !sg.abacEnabled {
return nil
}
@ -255,11 +251,16 @@ func (sg *SuperGraph) initAllowList() error {
var ac allow.Config
var err error
if !sg.conf.Production {
if len(sg.conf.AllowListFile) == 0 {
sg.conf.UseAllowList = false
sg.log.Printf("WRN allow list disabled no file specified")
}
if sg.conf.UseAllowList {
ac = allow.Config{CreateIfNotExists: true, Persist: true}
}
sg.allowList, err = allow.New(sg.conf.ConfigPathUsed(), ac)
sg.allowList, err = allow.New(sg.conf.AllowListFile, ac)
if err != nil {
return fmt.Errorf("failed to initialize allow list: %w", err)
}

5
core/resolve.go
View File

@ -6,7 +6,6 @@ import (
"net/http"
"strings"
"github.com/dosco/super-graph/config"
"github.com/dosco/super-graph/jsn"
)
@ -36,7 +35,7 @@ type resolvFn struct {
// }
// }
// func initRemotes(t config.Table) error {
// func initRemotes(t Table) error {
// h := xxhash.New()
// for _, r := range t.Remotes {
@ -92,7 +91,7 @@ type resolvFn struct {
// return nil
// }
func buildFn(r config.Remote) func(http.Header, []byte) ([]byte, error) {
func buildFn(r Remote) func(http.Header, []byte) ([]byte, error) {
reqURL := strings.Replace(r.URL, "$id", "%s", 1)
client := &http.Client{}