feat: url based multi-format loaders/decoders

This commit is contained in:
wpetit 2022-05-09 14:23:01 +02:00
parent 1353755683
commit 5383ed7ced
17 changed files with 375 additions and 4 deletions

1
go.mod
View File

@ -34,4 +34,5 @@ require (
github.com/go-chi/chi v1.5.4 github.com/go-chi/chi v1.5.4
github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.0.0 github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
) )

3
go.sum
View File

@ -94,8 +94,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -11,11 +11,16 @@ import (
"forge.cadoles.com/wpetit/formidable/internal/data" "forge.cadoles.com/wpetit/formidable/internal/data"
"forge.cadoles.com/wpetit/formidable/internal/data/format/hcl" "forge.cadoles.com/wpetit/formidable/internal/data/format/hcl"
"forge.cadoles.com/wpetit/formidable/internal/data/format/json" "forge.cadoles.com/wpetit/formidable/internal/data/format/json"
"forge.cadoles.com/wpetit/formidable/internal/data/format/yaml"
"forge.cadoles.com/wpetit/formidable/internal/data/scheme/file" "forge.cadoles.com/wpetit/formidable/internal/data/scheme/file"
"forge.cadoles.com/wpetit/formidable/internal/data/scheme/http"
"forge.cadoles.com/wpetit/formidable/internal/data/scheme/stdin"
"forge.cadoles.com/wpetit/formidable/internal/def" "forge.cadoles.com/wpetit/formidable/internal/def"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/santhosh-tekuri/jsonschema/v5" "github.com/santhosh-tekuri/jsonschema/v5"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
gohttp "net/http"
) )
const ( const (
@ -170,6 +175,8 @@ func outputWriter(ctx *cli.Context) (io.WriteCloser, error) {
func newLoader() *data.Loader { func newLoader() *data.Loader {
return data.NewLoader( return data.NewLoader(
file.NewLoaderHandler(), file.NewLoaderHandler(),
http.NewLoaderHandler(gohttp.DefaultClient),
stdin.NewLoaderHandler(),
) )
} }
@ -177,5 +184,6 @@ func newDecoder() *data.Decoder {
return data.NewDecoder( return data.NewDecoder(
json.NewDecoderHandler(), json.NewDecoderHandler(),
hcl.NewDecoderHandler(nil), hcl.NewDecoderHandler(nil),
yaml.NewDecoderHandler(),
) )
} }

View File

@ -7,8 +7,6 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const FormatQueryParam = "format"
type DecoderHandler interface { type DecoderHandler interface {
URLMatcher URLMatcher
Decode(url *url.URL, reader io.Reader) (interface{}, error) Decode(url *url.URL, reader io.Reader) (interface{}, error)

38
internal/data/encoder.go Normal file
View File

@ -0,0 +1,38 @@
package data
import (
"io"
"net/url"
"github.com/pkg/errors"
)
type EncoderHandler interface {
URLMatcher
Encode(url *url.URL, data interface{}) (io.Reader, error)
}
type Encoder struct {
handlers []EncoderHandler
}
func (e *Encoder) Encode(url *url.URL, data interface{}) (io.Reader, error) {
for _, h := range e.handlers {
if !h.Match(url) {
continue
}
reader, err := h.Encode(url, data)
if err != nil {
return nil, errors.WithStack(err)
}
return reader, nil
}
return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String())
}
func NewEncoder(handlers ...EncoderHandler) *Encoder {
return &Encoder{handlers}
}

View File

@ -0,0 +1,43 @@
package yaml
import (
"io"
"net/url"
"path"
"path/filepath"
"regexp"
"forge.cadoles.com/wpetit/formidable/internal/data/format"
"github.com/pkg/errors"
yaml "gopkg.in/yaml.v3"
)
var (
ExtensionYAML = regexp.MustCompile("\\.ya?ml$")
FormatYAML = "yaml"
)
type DecoderHandler struct{}
func (d *DecoderHandler) Match(url *url.URL) bool {
ext := filepath.Ext(path.Join(url.Host, url.Path))
return ExtensionYAML.MatchString(ext) ||
format.MatchURLQueryFormat(url, FormatYAML)
}
func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) {
decoder := yaml.NewDecoder(reader)
var values interface{}
if err := decoder.Decode(&values); err != nil {
return nil, errors.WithStack(err)
}
return values, nil
}
func NewDecoderHandler() *DecoderHandler {
return &DecoderHandler{}
}

View File

@ -0,0 +1,78 @@
package yaml
import (
"fmt"
"net/url"
"os"
"path/filepath"
"testing"
"github.com/pkg/errors"
)
type parserHandlerTestCase struct {
Path string
ExpectMatch bool
ExpectParseError bool
}
var parserHandlerTestCases = []parserHandlerTestCase{
{
Path: "testdata/dummy.yml",
ExpectMatch: true,
ExpectParseError: false,
},
{
Path: "file://testdata/dummy_no_ext?format=yaml",
ExpectMatch: true,
ExpectParseError: false,
},
}
func TestDecoderHandler(t *testing.T) {
t.Parallel()
handler := NewDecoderHandler()
for _, tc := range parserHandlerTestCases {
func(tc parserHandlerTestCase) {
t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) {
t.Parallel()
url, err := url.Parse(tc.Path)
if err != nil {
t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path))
}
if e, g := tc.ExpectMatch, handler.Match(url); e != g {
t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g)
}
if !tc.ExpectMatch {
return
}
cleanedPath := filepath.Join(url.Host, url.Path)
file, err := os.Open(cleanedPath)
if err != nil {
t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath))
}
defer func() {
if err := file.Close(); err != nil {
t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath))
}
}()
if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError {
t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path))
}
if tc.ExpectParseError {
t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
}
})
}(tc)
}
}

View File

@ -0,0 +1 @@
foo: bar

View File

@ -0,0 +1,2 @@
---
{}

View File

@ -10,7 +10,7 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const dummyFilePath = "testdata/dummy.txt" const dummyFilePath = "../testdata/dummy.txt"
var loaderHandlerTestCases []loaderHandlerTestCase var loaderHandlerTestCases []loaderHandlerTestCase

View File

@ -0,0 +1,39 @@
package http
import (
"io"
"net/http"
"net/url"
"github.com/pkg/errors"
)
const (
SchemeHTTP = "http"
SchemeHTTPS = "https"
)
type LoaderHandler struct {
client *http.Client
}
func (h *LoaderHandler) Match(url *url.URL) bool {
return url.Scheme == SchemeHTTP || url.Scheme == SchemeHTTPS
}
func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) {
res, err := h.client.Get(url.String())
if err != nil {
return nil, errors.WithStack(err)
}
if res.StatusCode != http.StatusOK {
return nil, errors.Errorf("unexpected status code '%d (%s)'", res.StatusCode, http.StatusText(res.StatusCode))
}
return res.Body, nil
}
func NewLoaderHandler(client *http.Client) *LoaderHandler {
return &LoaderHandler{client}
}

View File

@ -0,0 +1,87 @@
package http
import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/pkg/errors"
)
const (
testDataDir = "../testdata"
dummyPath = "dummy.txt"
)
type loaderHandlerTestCase struct {
URL string
ExpectMatch bool
ExpectOpenError bool
ExpectOpenContent string
}
func TestLoaderHandler(t *testing.T) {
t.Parallel()
staticHandler := http.FileServer(http.Dir(testDataDir))
server := httptest.NewServer(staticHandler)
defer server.Close()
loaderHandlerTestCases := []loaderHandlerTestCase{
{
URL: server.URL + "/" + dummyPath,
ExpectMatch: true,
ExpectOpenError: false,
ExpectOpenContent: "dummy",
},
}
handler := NewLoaderHandler(server.Client())
for _, tc := range loaderHandlerTestCases {
func(tc loaderHandlerTestCase) {
t.Run(fmt.Sprintf("Load '%s'", tc.URL), func(t *testing.T) {
url, err := url.Parse(tc.URL)
if err != nil {
t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.URL))
}
if e, g := tc.ExpectMatch, handler.Match(url); e != g {
t.Errorf("URL '%s': expected matching result '%v', got '%v'", tc.URL, e, g)
}
if !tc.ExpectMatch {
return
}
reader, err := handler.Open(url)
if err != nil && !tc.ExpectOpenError {
t.Fatal(errors.Wrapf(err, "could not open url '%s'", url.String()))
}
defer func() {
if err := reader.Close(); err != nil {
t.Error(errors.WithStack(err))
}
}()
if tc.ExpectOpenError {
t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
}
data, err := io.ReadAll(reader)
if err != nil {
t.Fatal(errors.WithStack(err))
}
if e, g := tc.ExpectOpenContent, string(data); e != g {
t.Errorf("URL '%s': expected content'%v', got '%v'", tc.URL, e, g)
}
})
}(tc)
}
}

View File

@ -0,0 +1,23 @@
package stdin
import (
"io"
"net/url"
"os"
)
const SchemeStdin = "stdin"
type LoaderHandler struct{}
func (h *LoaderHandler) Match(url *url.URL) bool {
return url.Scheme == SchemeStdin
}
func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) {
return os.Stdin, nil
}
func NewLoaderHandler() *LoaderHandler {
return &LoaderHandler{}
}

3
internal/data/url.go Normal file
View File

@ -0,0 +1,3 @@
package data
const FormatQueryParam = "format"

View File

@ -1,4 +1,4 @@
foo = { foo = {
bar = upper(totot) bar = "totot"
enabled = true enabled = true
} }

47
misc/script/install.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/sh
set -e
FORMIDABLE_RELEASES_URL="https://github.com/Bornholm/formidable/releases"
FORMIDABLE_DESTDIR="."
FORMIDABLE_FILE_BASENAME="frmd"
function main {
test -z "$FORMIDABLE_VERSION" && FORMIDABLE_VERSION="$(curl -sfL -o /dev/null -w %{url_effective} "$FORMIDABLE_RELEASES_URL/latest" |
rev |
cut -f1 -d'/'|
rev)"
# Check version variable initialization
test -z "$FORMIDABLE_VERSION" && {
echo "Unable to get Formidable version !" >&2
exit 1
}
test -z "$FORMIDABLE_TMPDIR" && FORMIDABLE_TMPDIR="$(mktemp -d)"
export TAR_FILE="$FORMIDABLE_TMPDIR/${FILE_BASENAME}_$(uname -s)_$(uname -m).tar.gz"
(
cd "$FORMIDABLE_TMPDIR"
# Download Formidable
echo "Downloading Formidable $FORMIDABLE_VERSION..."
curl -sfLo "$TAR_FILE" \
"$FORMIDABLE_RELEASES_URL/download/$FORMIDABLE_VERSION/${FORMIDABLE_FILE_BASENAME}_$(uname -s)_$(uname -m).tar.gz" ||
( echo "Error while downloading Formidable !" >&2 && exit 1 )
# Download checksums
curl -sfLo "checksums.txt" "$FORMIDABLE_RELEASES_URL/download/$FORMIDABLE_VERSION/checksums.txt"
echo "Verifying checksums..."
sha256sum --ignore-missing --quiet --check checksums.txt ||
( echo "Error while verifying checksums !" >&2 && exit 1 )
)
# Extracting archive files
tar -xf "$TAR_FILE" -C "$FORMIDABLE_TMPDIR"
# Moving downloaded binary to destination directory
mv -f "$FORMIDABLE_TMPDIR/$FORMIDABLE_FILE_BASENAME" "$FORMIDABLE_DESTDIR/"
}
main $@