From e05722cc2f5ae1cb21b3cad44de73eaee8d782f5 Mon Sep 17 00:00:00 2001 From: William Petit Date: Thu, 5 May 2022 16:22:52 +0200 Subject: [PATCH] feat: url based data loading system --- go.mod | 7 + go.sum | 47 +++++ internal/command/common.go | 91 ++++++---- internal/data/decoder.go | 40 +++++ internal/data/error.go | 5 + internal/data/format/hcl/decoder_handler.go | 165 ++++++++++++++++++ .../data/format/hcl/decoder_handler_test.go | 78 +++++++++ internal/data/format/hcl/testdata/dummy.hcl | 19 ++ .../data/format/hcl/testdata/dummy_no_ext | 1 + internal/data/format/json/decoder_handler.go | 42 +++++ .../data/format/json/decoder_handler_test.go | 78 +++++++++ internal/data/format/json/testdata/dummy.json | 3 + .../data/format/json/testdata/dummy_no_ext | 1 + internal/data/format/query.go | 7 + internal/data/loader.go | 38 ++++ internal/data/scheme/file/loader_handler.go | 31 ++++ .../data/scheme/file/loader_handler_test.go | 107 ++++++++++++ internal/data/scheme/file/testdata/dummy.txt | 1 + internal/data/url_matcher.go | 7 + misc/schema/hcl/my-schema.hcl | 34 ++++ misc/schema/hcl/my-schema.values.hcl | 4 + misc/schema/{ => json}/card.json | 0 misc/schema/{ => json}/filesystem.json | 0 .../schema/json/my-schema.json | 1 + .../schema/json/my-schema.values.json | 0 25 files changed, 774 insertions(+), 33 deletions(-) create mode 100644 internal/data/decoder.go create mode 100644 internal/data/error.go create mode 100644 internal/data/format/hcl/decoder_handler.go create mode 100644 internal/data/format/hcl/decoder_handler_test.go create mode 100644 internal/data/format/hcl/testdata/dummy.hcl create mode 100644 internal/data/format/hcl/testdata/dummy_no_ext create mode 100644 internal/data/format/json/decoder_handler.go create mode 100644 internal/data/format/json/decoder_handler_test.go create mode 100644 internal/data/format/json/testdata/dummy.json create mode 100644 internal/data/format/json/testdata/dummy_no_ext create mode 100644 internal/data/format/query.go create mode 100644 internal/data/loader.go create mode 100644 internal/data/scheme/file/loader_handler.go create mode 100644 internal/data/scheme/file/loader_handler_test.go create mode 100644 internal/data/scheme/file/testdata/dummy.txt create mode 100644 internal/data/url_matcher.go create mode 100644 misc/schema/hcl/my-schema.hcl create mode 100644 misc/schema/hcl/my-schema.values.hcl rename misc/schema/{ => json}/card.json (100%) rename misc/schema/{ => json}/filesystem.json (100%) rename schema.json => misc/schema/json/my-schema.json (97%) rename values.json => misc/schema/json/my-schema.values.json (100%) diff --git a/go.mod b/go.mod index 51d8d31..d68bfc1 100644 --- a/go.mod +++ b/go.mod @@ -4,21 +4,28 @@ go 1.18 require ( github.com/Masterminds/sprig/v3 v3.2.2 + github.com/hashicorp/hcl/v2 v2.12.0 github.com/pkg/errors v0.9.1 github.com/urfave/cli/v2 v2.4.0 + github.com/zclconf/go-cty v1.8.0 ) require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/agext/levenshtein v1.2.1 // indirect + github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect + github.com/google/go-cmp v0.3.1 // indirect github.com/google/uuid v1.1.1 // indirect github.com/huandu/xstrings v1.3.1 // indirect github.com/imdario/mergo v0.3.11 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 // indirect + golang.org/x/text v0.3.5 // indirect ) require ( diff --git a/go.sum b/go.sum index 8d8eaa1..5537bb9 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,12 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= +github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= +github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM= +github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= +github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= +github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -12,14 +18,32 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/hcl/v2 v2.12.0 h1:PsYxySWpMD4KPaoJLnsHwtK5Qptvj/4Q6s0t4sUxZf4= +github.com/hashicorp/hcl/v2 v2.12.0/go.mod h1:FwWsfWEjyV/CMj8s/gqAuiviY72rJ1/oayI9WftqcKg= github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= +github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -30,24 +54,47 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 h1:TToq11gyfNlrMFZiYujSekIsPd9AmsA2Bj/iv+s4JHE= github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= +github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8= +github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= +github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904 h1:bXoxMPcSLOq08zI3/c5dEBT6lE4eh+jOh886GHrn6V8= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= diff --git a/internal/command/common.go b/internal/command/common.go index c0e10a1..9147a92 100644 --- a/internal/command/common.go +++ b/internal/command/common.go @@ -1,11 +1,17 @@ package command import ( - "encoding/json" + "bytes" "io" + "net/url" "os" - "strings" + encjson "encoding/json" + + "forge.cadoles.com/wpetit/formidable/internal/data" + "forge.cadoles.com/wpetit/formidable/internal/data/format/hcl" + "forge.cadoles.com/wpetit/formidable/internal/data/format/json" + "forge.cadoles.com/wpetit/formidable/internal/data/scheme/file" "forge.cadoles.com/wpetit/formidable/internal/def" "github.com/pkg/errors" "github.com/santhosh-tekuri/jsonschema/v5" @@ -22,13 +28,13 @@ func commonFlags() []cli.Flag { Name: "defaults", Aliases: []string{"d"}, Usage: "Default values as JSON or file path prefixed by '@'", - Value: "{}", + Value: "", }, &cli.StringFlag{ Name: "values", Aliases: []string{"v"}, Usage: "Current values as JSON or file path prefixed by '@'", - Value: "{}", + Value: "", }, &cli.StringFlag{ Name: "schema", @@ -45,49 +51,43 @@ func commonFlags() []cli.Flag { } } -func loadJSONFlag(ctx *cli.Context, flagName string) (interface{}, error) { +func loadURLFlag(ctx *cli.Context, flagName string) (interface{}, error) { flagValue := ctx.String(flagName) if flagValue == "" { return nil, nil } - if !strings.HasPrefix(flagValue, filePathPrefix) { - var value interface{} + loader := newLoader() - if err := json.Unmarshal([]byte(flagValue), &value); err != nil { - return nil, errors.WithStack(err) - } - - return value, nil + url, err := url.Parse(flagValue) + if err != nil { + return nil, errors.WithStack(err) } - flagValue = strings.TrimPrefix(flagValue, filePathPrefix) - - file, err := os.Open(flagValue) + reader, err := loader.Open(url) if err != nil { return nil, errors.WithStack(err) } defer func() { - if err := file.Close(); err != nil { + if err := reader.Close(); err != nil { panic(errors.WithStack(err)) } }() - reader := json.NewDecoder(file) + decoder := newDecoder() - var values interface{} - - if err := reader.Decode(&values); err != nil { + data, err := decoder.Decode(url, reader) + if err != nil { return nil, errors.WithStack(err) } - return values, nil + return data, nil } func loadValues(ctx *cli.Context) (interface{}, error) { - values, err := loadJSONFlag(ctx, "values") + values, err := loadURLFlag(ctx, "values") if err != nil { return nil, errors.WithStack(err) } @@ -96,33 +96,45 @@ func loadValues(ctx *cli.Context) (interface{}, error) { } func loadDefaults(ctx *cli.Context) (interface{}, error) { - values, err := loadJSONFlag(ctx, "defaults") + defaults, err := loadURLFlag(ctx, "defaults") if err != nil { return nil, errors.WithStack(err) } - return values, nil + return defaults, nil } func loadSchema(ctx *cli.Context) (*jsonschema.Schema, error) { schemaFlag := ctx.String("schema") + if schemaFlag == "" { + return def.Schema, nil + } + + schemaTree, err := loadURLFlag(ctx, "schema") + if err != nil { + return nil, errors.WithStack(err) + } + + // Reencode schema to JSON format + var buf bytes.Buffer + encoder := encjson.NewEncoder(&buf) + + if err := encoder.Encode(schemaTree); err != nil { + return nil, errors.WithStack(err) + } + compiler := jsonschema.NewCompiler() compiler.ExtractAnnotations = true compiler.AssertFormat = true compiler.AssertContent = true - var ( - schema *jsonschema.Schema - err error - ) - - if schemaFlag == "" { - schema = def.Schema - } else { - schema, err = compiler.Compile(schemaFlag) + if err := compiler.AddResource(schemaFlag, &buf); err != nil { + return nil, errors.WithStack(err) } + + schema, err := compiler.Compile(schemaFlag) if err != nil { return nil, errors.WithStack(err) } @@ -154,3 +166,16 @@ func outputWriter(ctx *cli.Context) (io.WriteCloser, error) { return file, nil } + +func newLoader() *data.Loader { + return data.NewLoader( + file.NewLoaderHandler(), + ) +} + +func newDecoder() *data.Decoder { + return data.NewDecoder( + json.NewDecoderHandler(), + hcl.NewDecoderHandler(nil), + ) +} diff --git a/internal/data/decoder.go b/internal/data/decoder.go new file mode 100644 index 0000000..6e0f7e6 --- /dev/null +++ b/internal/data/decoder.go @@ -0,0 +1,40 @@ +package data + +import ( + "io" + "net/url" + + "github.com/pkg/errors" +) + +const FormatQueryParam = "format" + +type DecoderHandler interface { + URLMatcher + Decode(url *url.URL, reader io.Reader) (interface{}, error) +} + +type Decoder struct { + handlers []DecoderHandler +} + +func (d *Decoder) Decode(url *url.URL, reader io.ReadCloser) (interface{}, error) { + for _, h := range d.handlers { + if !h.Match(url) { + continue + } + + data, err := h.Decode(url, reader) + if err != nil { + return nil, errors.WithStack(err) + } + + return data, nil + } + + return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String()) +} + +func NewDecoder(handlers ...DecoderHandler) *Decoder { + return &Decoder{handlers} +} diff --git a/internal/data/error.go b/internal/data/error.go new file mode 100644 index 0000000..4d57aae --- /dev/null +++ b/internal/data/error.go @@ -0,0 +1,5 @@ +package data + +import "github.com/pkg/errors" + +var ErrHandlerNotFound = errors.New("handler not found") diff --git a/internal/data/format/hcl/decoder_handler.go b/internal/data/format/hcl/decoder_handler.go new file mode 100644 index 0000000..7fdbec1 --- /dev/null +++ b/internal/data/format/hcl/decoder_handler.go @@ -0,0 +1,165 @@ +package hcl + +import ( + "io" + "net/url" + "path" + "path/filepath" + + "forge.cadoles.com/wpetit/formidable/internal/data/format" + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/pkg/errors" + "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/function" + "github.com/zclconf/go-cty/cty/gocty" +) + +const ( + ExtensionHCL = ".hcl" + FormatHCL = "hcl" +) + +type DecoderHandler struct { + ctx *hcl.EvalContext +} + +func (d *DecoderHandler) Match(url *url.URL) bool { + ext := filepath.Ext(path.Join(url.Host, url.Path)) + + return ext == ExtensionHCL || + format.MatchURLQueryFormat(url, FormatHCL) +} + +func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) { + data, err := io.ReadAll(reader) + if err != nil { + return nil, errors.WithStack(err) + } + + ctx := d.ctx + if ctx == nil { + ctx = &hcl.EvalContext{ + Variables: make(map[string]cty.Value), + Functions: make(map[string]function.Function), + } + } + + parser := hclparse.NewParser() + + file, diags := parser.ParseHCL(data, url.String()) + if diags.HasErrors() { + return nil, errors.WithStack(diags) + } + + var tree map[string]interface{} + + diags = gohcl.DecodeBody(file.Body, ctx, &tree) + if diags.HasErrors() { + return nil, errors.WithStack(diags) + } + + ctx = ctx.NewChild() + + values, err := hclTreeToRawValues(ctx, tree) + if err != nil { + return nil, errors.WithStack(err) + } + + return values, nil +} + +func NewDecoderHandler(ctx *hcl.EvalContext) *DecoderHandler { + return &DecoderHandler{ctx} +} + +func hclTreeToRawValues(ctx *hcl.EvalContext, tree map[string]interface{}) (map[string]interface{}, error) { + values := make(map[string]interface{}) + + for key, branch := range tree { + v, err := hclBranchToRawValue(ctx, branch) + if err != nil { + return nil, errors.WithStack(err) + } + + values[key] = v + } + + return values, nil +} + +func hclBranchToRawValue(ctx *hcl.EvalContext, branch interface{}) (interface{}, error) { + switch typ := branch.(type) { + case *hcl.Attribute: + val, diags := typ.Expr.Value(ctx) + if diags.HasErrors() { + return nil, errors.WithStack(diags) + } + + raw, err := ctyValueToRaw(ctx, val) + if err != nil { + return nil, errors.WithStack(err) + } + + return raw, nil + default: + return nil, errors.Errorf("unexpected type '%T'", typ) + } +} + +func ctyValueToRaw(ctx *hcl.EvalContext, val cty.Value) (interface{}, error) { + if val.Type().Equals(cty.Bool) { + var raw bool + + if err := gocty.FromCtyValue(val, &raw); err != nil { + return nil, errors.WithStack(err) + } + + return raw, nil + } else if val.Type().Equals(cty.Number) { + var raw float64 + + if err := gocty.FromCtyValue(val, &raw); err != nil { + return nil, errors.WithStack(err) + } + + return raw, nil + } else if val.Type().Equals(cty.String) { + var raw string + + if err := gocty.FromCtyValue(val, &raw); err != nil { + return nil, errors.WithStack(err) + } + + return raw, nil + } else if val.Type().IsObjectType() { + obj := make(map[string]interface{}) + + for k, v := range val.AsValueMap() { + rv, err := ctyValueToRaw(ctx, v) + if err != nil { + return nil, errors.WithStack(err) + } + + obj[k] = rv + } + + return obj, nil + } else if val.Type().IsTupleType() { + sl := make([]interface{}, 0) + + for _, v := range val.AsValueSlice() { + rv, err := ctyValueToRaw(ctx, v) + if err != nil { + return nil, errors.WithStack(err) + } + + sl = append(sl, rv) + } + + return sl, nil + } + + return nil, errors.Errorf("unexpected cty.Type '%s'", val.Type().FriendlyName()) +} diff --git a/internal/data/format/hcl/decoder_handler_test.go b/internal/data/format/hcl/decoder_handler_test.go new file mode 100644 index 0000000..14ea41c --- /dev/null +++ b/internal/data/format/hcl/decoder_handler_test.go @@ -0,0 +1,78 @@ +package hcl + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" +) + +type parserHandlerTestCase struct { + Path string + ExpectMatch bool + ExpectParseError bool +} + +var parserHandlerTestCases = []parserHandlerTestCase{ + { + Path: "testdata/dummy.hcl", + ExpectMatch: true, + ExpectParseError: false, + }, + { + Path: "file://testdata/dummy_no_ext?format=hcl", + ExpectMatch: true, + ExpectParseError: false, + }, +} + +func TestDecoderHandler(t *testing.T) { + t.Parallel() + + handler := NewDecoderHandler(nil) + + for _, tc := range parserHandlerTestCases { + func(tc parserHandlerTestCase) { + t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) { + t.Parallel() + + url, err := url.Parse(tc.Path) + if err != nil { + t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path)) + } + + if e, g := tc.ExpectMatch, handler.Match(url); e != g { + t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g) + } + + if !tc.ExpectMatch { + return + } + + cleanedPath := filepath.Join(url.Host, url.Path) + + file, err := os.Open(cleanedPath) + if err != nil { + t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath)) + } + + defer func() { + if err := file.Close(); err != nil { + t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath)) + } + }() + + if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError { + t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path)) + } + + if tc.ExpectParseError { + t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String())) + } + }) + }(tc) + } +} diff --git a/internal/data/format/hcl/testdata/dummy.hcl b/internal/data/format/hcl/testdata/dummy.hcl new file mode 100644 index 0000000..99a2d7f --- /dev/null +++ b/internal/data/format/hcl/testdata/dummy.hcl @@ -0,0 +1,19 @@ +test = 1 + +test1 = 2 + 1 + +foo = { + description = "Ça fait des trucs" + type = "object" + properties = { + type = "string" + minLength = 5 + } + test = [ + "foo", + { + test = "foo" + }, + 5 + 5.2 + ] +} \ No newline at end of file diff --git a/internal/data/format/hcl/testdata/dummy_no_ext b/internal/data/format/hcl/testdata/dummy_no_ext new file mode 100644 index 0000000..21731b3 --- /dev/null +++ b/internal/data/format/hcl/testdata/dummy_no_ext @@ -0,0 +1 @@ +foo = "bar" \ No newline at end of file diff --git a/internal/data/format/json/decoder_handler.go b/internal/data/format/json/decoder_handler.go new file mode 100644 index 0000000..fe78957 --- /dev/null +++ b/internal/data/format/json/decoder_handler.go @@ -0,0 +1,42 @@ +package json + +import ( + "encoding/json" + "io" + "net/url" + "path" + "path/filepath" + + "forge.cadoles.com/wpetit/formidable/internal/data/format" + "github.com/pkg/errors" +) + +const ( + ExtensionJSON = ".json" + FormatJSON = "json" +) + +type DecoderHandler struct{} + +func (d *DecoderHandler) Match(url *url.URL) bool { + ext := filepath.Ext(path.Join(url.Host, url.Path)) + + return ext == ExtensionJSON || + format.MatchURLQueryFormat(url, FormatJSON) +} + +func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) { + decoder := json.NewDecoder(reader) + + var values interface{} + + if err := decoder.Decode(&values); err != nil { + return nil, errors.WithStack(err) + } + + return values, nil +} + +func NewDecoderHandler() *DecoderHandler { + return &DecoderHandler{} +} diff --git a/internal/data/format/json/decoder_handler_test.go b/internal/data/format/json/decoder_handler_test.go new file mode 100644 index 0000000..819ed3f --- /dev/null +++ b/internal/data/format/json/decoder_handler_test.go @@ -0,0 +1,78 @@ +package json + +import ( + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/pkg/errors" +) + +type parserHandlerTestCase struct { + Path string + ExpectMatch bool + ExpectParseError bool +} + +var parserHandlerTestCases = []parserHandlerTestCase{ + { + Path: "testdata/dummy.json", + ExpectMatch: true, + ExpectParseError: false, + }, + { + Path: "file://testdata/dummy_no_ext?format=json", + ExpectMatch: true, + ExpectParseError: false, + }, +} + +func TestDecoderHandler(t *testing.T) { + t.Parallel() + + handler := NewDecoderHandler() + + for _, tc := range parserHandlerTestCases { + func(tc parserHandlerTestCase) { + t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) { + t.Parallel() + + url, err := url.Parse(tc.Path) + if err != nil { + t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path)) + } + + if e, g := tc.ExpectMatch, handler.Match(url); e != g { + t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g) + } + + if !tc.ExpectMatch { + return + } + + cleanedPath := filepath.Join(url.Host, url.Path) + + file, err := os.Open(cleanedPath) + if err != nil { + t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath)) + } + + defer func() { + if err := file.Close(); err != nil { + t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath)) + } + }() + + if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError { + t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path)) + } + + if tc.ExpectParseError { + t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String())) + } + }) + }(tc) + } +} diff --git a/internal/data/format/json/testdata/dummy.json b/internal/data/format/json/testdata/dummy.json new file mode 100644 index 0000000..8a79687 --- /dev/null +++ b/internal/data/format/json/testdata/dummy.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} \ No newline at end of file diff --git a/internal/data/format/json/testdata/dummy_no_ext b/internal/data/format/json/testdata/dummy_no_ext new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/internal/data/format/json/testdata/dummy_no_ext @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/internal/data/format/query.go b/internal/data/format/query.go new file mode 100644 index 0000000..6822ef7 --- /dev/null +++ b/internal/data/format/query.go @@ -0,0 +1,7 @@ +package format + +import "net/url" + +func MatchURLQueryFormat(url *url.URL, format string) bool { + return url.Query().Get("format") == format +} diff --git a/internal/data/loader.go b/internal/data/loader.go new file mode 100644 index 0000000..a99a1cd --- /dev/null +++ b/internal/data/loader.go @@ -0,0 +1,38 @@ +package data + +import ( + "io" + "net/url" + + "github.com/pkg/errors" +) + +type LoaderHandler interface { + URLMatcher + Open(url *url.URL) (io.ReadCloser, error) +} + +type Loader struct { + handlers []LoaderHandler +} + +func (l *Loader) Open(url *url.URL) (io.ReadCloser, error) { + for _, h := range l.handlers { + if !h.Match(url) { + continue + } + + reader, err := h.Open(url) + if err != nil { + return nil, errors.WithStack(err) + } + + return reader, nil + } + + return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String()) +} + +func NewLoader(handlers ...LoaderHandler) *Loader { + return &Loader{handlers} +} diff --git a/internal/data/scheme/file/loader_handler.go b/internal/data/scheme/file/loader_handler.go new file mode 100644 index 0000000..d1fcec0 --- /dev/null +++ b/internal/data/scheme/file/loader_handler.go @@ -0,0 +1,31 @@ +package file + +import ( + "io" + "net/url" + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +const SchemeFile = "file" + +type LoaderHandler struct{} + +func (h *LoaderHandler) Match(url *url.URL) bool { + return url.Scheme == SchemeFile || url.Scheme == "" +} + +func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) { + file, err := os.Open(filepath.Join(url.Host, url.Path)) + if err != nil { + return nil, errors.Wrapf(err, "could not open file '%s'", url.Path) + } + + return file, nil +} + +func NewLoaderHandler() *LoaderHandler { + return &LoaderHandler{} +} diff --git a/internal/data/scheme/file/loader_handler_test.go b/internal/data/scheme/file/loader_handler_test.go new file mode 100644 index 0000000..59e4295 --- /dev/null +++ b/internal/data/scheme/file/loader_handler_test.go @@ -0,0 +1,107 @@ +package file + +import ( + "fmt" + "io" + "net/url" + "path/filepath" + "testing" + + "github.com/pkg/errors" +) + +const dummyFilePath = "testdata/dummy.txt" + +var loaderHandlerTestCases []loaderHandlerTestCase + +func init() { + dummyFileAbsolutePath, err := filepath.Abs(dummyFilePath) + if err != nil { + panic(errors.WithStack(err)) + } + + loaderHandlerTestCases = []loaderHandlerTestCase{ + { + URL: SchemeFile + "://" + dummyFilePath, + ExpectMatch: true, + ExpectOpenError: false, + ExpectOpenContent: "dummy", + }, + { + URL: dummyFilePath, + ExpectMatch: true, + ExpectOpenError: false, + ExpectOpenContent: "dummy", + }, + { + URL: dummyFileAbsolutePath, + ExpectMatch: true, + ExpectOpenError: false, + ExpectOpenContent: "dummy", + }, + { + URL: SchemeFile + "://" + dummyFileAbsolutePath, + ExpectMatch: true, + ExpectOpenError: false, + ExpectOpenContent: "dummy", + }, + } +} + +type loaderHandlerTestCase struct { + URL string + ExpectMatch bool + ExpectOpenError bool + ExpectOpenContent string +} + +func TestLoaderHandler(t *testing.T) { + t.Parallel() + + handler := NewLoaderHandler() + + for _, tc := range loaderHandlerTestCases { + func(tc loaderHandlerTestCase) { + t.Run(fmt.Sprintf("Load '%s'", tc.URL), func(t *testing.T) { + t.Parallel() + + url, err := url.Parse(tc.URL) + if err != nil { + t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.URL)) + } + + if e, g := tc.ExpectMatch, handler.Match(url); e != g { + t.Errorf("URL '%s': expected matching result '%v', got '%v'", tc.URL, e, g) + } + + if !tc.ExpectMatch { + return + } + + reader, err := handler.Open(url) + if err != nil && !tc.ExpectOpenError { + t.Fatal(errors.Wrapf(err, "could not open url '%s'", url.String())) + } + + defer func() { + if err := reader.Close(); err != nil { + t.Error(errors.WithStack(err)) + } + }() + + if tc.ExpectOpenError { + t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String())) + } + + data, err := io.ReadAll(reader) + if err != nil { + t.Fatal(errors.WithStack(err)) + } + + if e, g := tc.ExpectOpenContent, string(data); e != g { + t.Errorf("URL '%s': expected content'%v', got '%v'", tc.URL, e, g) + } + }) + }(tc) + } +} diff --git a/internal/data/scheme/file/testdata/dummy.txt b/internal/data/scheme/file/testdata/dummy.txt new file mode 100644 index 0000000..2995a4d --- /dev/null +++ b/internal/data/scheme/file/testdata/dummy.txt @@ -0,0 +1 @@ +dummy \ No newline at end of file diff --git a/internal/data/url_matcher.go b/internal/data/url_matcher.go new file mode 100644 index 0000000..9a8db13 --- /dev/null +++ b/internal/data/url_matcher.go @@ -0,0 +1,7 @@ +package data + +import "net/url" + +type URLMatcher interface { + Match(url *url.URL) bool +} diff --git a/misc/schema/hcl/my-schema.hcl b/misc/schema/hcl/my-schema.hcl new file mode 100644 index 0000000..81422e3 --- /dev/null +++ b/misc/schema/hcl/my-schema.hcl @@ -0,0 +1,34 @@ +title = "My schema" +description = "Test" +type = "object" +required = [ "foo" ] +properties = { + foo = { + description = "Ça fait des trucs" + type = "object" + properties = { + bar = { + type = "string" + minLength = 5 + } + enabled = { + type = "boolean" + } + myItems = { + type = "array" + items = { + type = "object" + properties = { + stringProp = { + type = "string" + minLength = 10 + } + numericProp = { + type = "integer" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/misc/schema/hcl/my-schema.values.hcl b/misc/schema/hcl/my-schema.values.hcl new file mode 100644 index 0000000..3f484a1 --- /dev/null +++ b/misc/schema/hcl/my-schema.values.hcl @@ -0,0 +1,4 @@ +foo = { + bar = upper(totot) + enabled = true +} diff --git a/misc/schema/card.json b/misc/schema/json/card.json similarity index 100% rename from misc/schema/card.json rename to misc/schema/json/card.json diff --git a/misc/schema/filesystem.json b/misc/schema/json/filesystem.json similarity index 100% rename from misc/schema/filesystem.json rename to misc/schema/json/filesystem.json diff --git a/schema.json b/misc/schema/json/my-schema.json similarity index 97% rename from schema.json rename to misc/schema/json/my-schema.json index 3cc5a0e..8eacc43 100644 --- a/schema.json +++ b/misc/schema/json/my-schema.json @@ -1,6 +1,7 @@ { "$id": "https://example.com/custom.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "My Schema", "description": "Test", "type": "object", "required": [ diff --git a/values.json b/misc/schema/json/my-schema.values.json similarity index 100% rename from values.json rename to misc/schema/json/my-schema.values.json