feat: url based multi-format loaders/decoders

go.mod

@@ -34,4 +34,5 @@ require (
 	github.com/go-chi/chi v1.5.4
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.0.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -94,8 +94,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/command/common.go

@@ -11,11 +11,16 @@ import (
 	"forge.cadoles.com/wpetit/formidable/internal/data"
 	"forge.cadoles.com/wpetit/formidable/internal/data/format/hcl"
 	"forge.cadoles.com/wpetit/formidable/internal/data/format/json"
+	"forge.cadoles.com/wpetit/formidable/internal/data/format/yaml"
 	"forge.cadoles.com/wpetit/formidable/internal/data/scheme/file"
+	"forge.cadoles.com/wpetit/formidable/internal/data/scheme/http"
+	"forge.cadoles.com/wpetit/formidable/internal/data/scheme/stdin"
 	"forge.cadoles.com/wpetit/formidable/internal/def"
 	"github.com/pkg/errors"
 	"github.com/santhosh-tekuri/jsonschema/v5"
 	"github.com/urfave/cli/v2"
+
+	gohttp "net/http"
 )
 
 const (
@@ -170,6 +175,8 @@ func outputWriter(ctx *cli.Context) (io.WriteCloser, error) {
 func newLoader() *data.Loader {
 	return data.NewLoader(
 		file.NewLoaderHandler(),
+		http.NewLoaderHandler(gohttp.DefaultClient),
+		stdin.NewLoaderHandler(),
 	)
 }
 
@@ -177,5 +184,6 @@ func newDecoder() *data.Decoder {
 	return data.NewDecoder(
 		json.NewDecoderHandler(),
 		hcl.NewDecoderHandler(nil),
+		yaml.NewDecoderHandler(),
 	)
 }

internal/data/decoder.go

@@ -7,8 +7,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-const FormatQueryParam = "format"
-
 type DecoderHandler interface {
 	URLMatcher
 	Decode(url *url.URL, reader io.Reader) (interface{}, error)

internal/data/encoder.go

@@ -0,0 +1,38 @@
+package data
+
+import (
+	"io"
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+type EncoderHandler interface {
+	URLMatcher
+	Encode(url *url.URL, data interface{}) (io.Reader, error)
+}
+
+type Encoder struct {
+	handlers []EncoderHandler
+}
+
+func (e *Encoder) Encode(url *url.URL, data interface{}) (io.Reader, error) {
+	for _, h := range e.handlers {
+		if !h.Match(url) {
+			continue
+		}
+
+		reader, err := h.Encode(url, data)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return reader, nil
+	}
+
+	return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String())
+}
+
+func NewEncoder(handlers ...EncoderHandler) *Encoder {
+	return &Encoder{handlers}
+}

internal/data/format/yaml/decoder_handler.go

@@ -0,0 +1,43 @@
+package yaml
+
+import (
+	"io"
+	"net/url"
+	"path"
+	"path/filepath"
+	"regexp"
+
+	"forge.cadoles.com/wpetit/formidable/internal/data/format"
+	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v3"
+)
+
+var (
+	ExtensionYAML = regexp.MustCompile("\\.ya?ml$")
+	FormatYAML    = "yaml"
+)
+
+type DecoderHandler struct{}
+
+func (d *DecoderHandler) Match(url *url.URL) bool {
+	ext := filepath.Ext(path.Join(url.Host, url.Path))
+
+	return ExtensionYAML.MatchString(ext) ||
+		format.MatchURLQueryFormat(url, FormatYAML)
+}
+
+func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) {
+	decoder := yaml.NewDecoder(reader)
+
+	var values interface{}
+
+	if err := decoder.Decode(&values); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return values, nil
+}
+
+func NewDecoderHandler() *DecoderHandler {
+	return &DecoderHandler{}
+}

internal/data/format/yaml/decoder_handler_test.go

@@ -0,0 +1,78 @@
+package yaml
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+type parserHandlerTestCase struct {
+	Path             string
+	ExpectMatch      bool
+	ExpectParseError bool
+}
+
+var parserHandlerTestCases = []parserHandlerTestCase{
+	{
+		Path:             "testdata/dummy.yml",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+	{
+		Path:             "file://testdata/dummy_no_ext?format=yaml",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+}
+
+func TestDecoderHandler(t *testing.T) {
+	t.Parallel()
+
+	handler := NewDecoderHandler()
+
+	for _, tc := range parserHandlerTestCases {
+		func(tc parserHandlerTestCase) {
+			t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) {
+				t.Parallel()
+
+				url, err := url.Parse(tc.Path)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path))
+				}
+
+				if e, g := tc.ExpectMatch, handler.Match(url); e != g {
+					t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g)
+				}
+
+				if !tc.ExpectMatch {
+					return
+				}
+
+				cleanedPath := filepath.Join(url.Host, url.Path)
+
+				file, err := os.Open(cleanedPath)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath))
+				}
+
+				defer func() {
+					if err := file.Close(); err != nil {
+						t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath))
+					}
+				}()
+
+				if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError {
+					t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path))
+				}
+
+				if tc.ExpectParseError {
+					t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
+				}
+			})
+		}(tc)
+	}
+}

internal/data/format/yaml/testdata/dummy.yml

@@ -0,0 +1 @@
+foo: bar

internal/data/format/yaml/testdata/dummy_no_ext

@@ -0,0 +1,2 @@
+---
+{}

internal/data/scheme/file/loader_handler_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-const dummyFilePath = "testdata/dummy.txt"
+const dummyFilePath = "../testdata/dummy.txt"
 
 var loaderHandlerTestCases []loaderHandlerTestCase
 

internal/data/scheme/http/loader_handler.go

@@ -0,0 +1,39 @@
+package http
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	SchemeHTTP  = "http"
+	SchemeHTTPS = "https"
+)
+
+type LoaderHandler struct {
+	client *http.Client
+}
+
+func (h *LoaderHandler) Match(url *url.URL) bool {
+	return url.Scheme == SchemeHTTP || url.Scheme == SchemeHTTPS
+}
+
+func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) {
+	res, err := h.client.Get(url.String())
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	if res.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("unexpected status code '%d (%s)'", res.StatusCode, http.StatusText(res.StatusCode))
+	}
+
+	return res.Body, nil
+}
+
+func NewLoaderHandler(client *http.Client) *LoaderHandler {
+	return &LoaderHandler{client}
+}

internal/data/scheme/http/loader_handler_test.go

@@ -0,0 +1,87 @@
+package http
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	testDataDir = "../testdata"
+	dummyPath   = "dummy.txt"
+)
+
+type loaderHandlerTestCase struct {
+	URL               string
+	ExpectMatch       bool
+	ExpectOpenError   bool
+	ExpectOpenContent string
+}
+
+func TestLoaderHandler(t *testing.T) {
+	t.Parallel()
+
+	staticHandler := http.FileServer(http.Dir(testDataDir))
+
+	server := httptest.NewServer(staticHandler)
+	defer server.Close()
+
+	loaderHandlerTestCases := []loaderHandlerTestCase{
+		{
+			URL:               server.URL + "/" + dummyPath,
+			ExpectMatch:       true,
+			ExpectOpenError:   false,
+			ExpectOpenContent: "dummy",
+		},
+	}
+
+	handler := NewLoaderHandler(server.Client())
+
+	for _, tc := range loaderHandlerTestCases {
+		func(tc loaderHandlerTestCase) {
+			t.Run(fmt.Sprintf("Load '%s'", tc.URL), func(t *testing.T) {
+				url, err := url.Parse(tc.URL)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.URL))
+				}
+
+				if e, g := tc.ExpectMatch, handler.Match(url); e != g {
+					t.Errorf("URL '%s': expected matching result '%v', got '%v'", tc.URL, e, g)
+				}
+
+				if !tc.ExpectMatch {
+					return
+				}
+
+				reader, err := handler.Open(url)
+				if err != nil && !tc.ExpectOpenError {
+					t.Fatal(errors.Wrapf(err, "could not open url '%s'", url.String()))
+				}
+
+				defer func() {
+					if err := reader.Close(); err != nil {
+						t.Error(errors.WithStack(err))
+					}
+				}()
+
+				if tc.ExpectOpenError {
+					t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
+				}
+
+				data, err := io.ReadAll(reader)
+				if err != nil {
+					t.Fatal(errors.WithStack(err))
+				}
+
+				if e, g := tc.ExpectOpenContent, string(data); e != g {
+					t.Errorf("URL '%s': expected content'%v', got '%v'", tc.URL, e, g)
+				}
+			})
+		}(tc)
+	}
+}

internal/data/scheme/stdin/loader_handler.go

@@ -0,0 +1,23 @@
+package stdin
+
+import (
+	"io"
+	"net/url"
+	"os"
+)
+
+const SchemeStdin = "stdin"
+
+type LoaderHandler struct{}
+
+func (h *LoaderHandler) Match(url *url.URL) bool {
+	return url.Scheme == SchemeStdin
+}
+
+func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) {
+	return os.Stdin, nil
+}
+
+func NewLoaderHandler() *LoaderHandler {
+	return &LoaderHandler{}
+}

internal/data/url.go

@@ -0,0 +1,3 @@
+package data
+
+const FormatQueryParam = "format"

misc/schema/hcl/my-schema.values.hcl

@@ -1,4 +1,4 @@
 foo = {
-   bar = upper(totot)
+   bar = "totot"
    enabled = true
 }

misc/script/install.sh

@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e
+
+FORMIDABLE_RELEASES_URL="https://github.com/Bornholm/formidable/releases"
+FORMIDABLE_DESTDIR="."
+FORMIDABLE_FILE_BASENAME="frmd"
+
+function main {
+    test -z "$FORMIDABLE_VERSION" && FORMIDABLE_VERSION="$(curl -sfL -o /dev/null -w %{url_effective} "$FORMIDABLE_RELEASES_URL/latest" |
+        rev |
+        cut -f1 -d'/'|
+        rev)"
+
+    # Check version variable initialization
+    test -z "$FORMIDABLE_VERSION" && {
+        echo "Unable to get Formidable version !" >&2
+        exit 1
+    }
+
+    test -z "$FORMIDABLE_TMPDIR" && FORMIDABLE_TMPDIR="$(mktemp -d)"
+    export TAR_FILE="$FORMIDABLE_TMPDIR/${FILE_BASENAME}_$(uname -s)_$(uname -m).tar.gz"
+
+    (
+        cd "$FORMIDABLE_TMPDIR"
+
+        # Download Formidable
+        echo "Downloading Formidable $FORMIDABLE_VERSION..."
+        curl -sfLo "$TAR_FILE" \
+            "$FORMIDABLE_RELEASES_URL/download/$FORMIDABLE_VERSION/${FORMIDABLE_FILE_BASENAME}_$(uname -s)_$(uname -m).tar.gz" ||
+            ( echo  "Error while downloading Formidable !" >&2 && exit 1 )
+        
+        # Download checksums
+        curl -sfLo "checksums.txt" "$FORMIDABLE_RELEASES_URL/download/$FORMIDABLE_VERSION/checksums.txt"
+        
+        echo "Verifying checksums..."
+        sha256sum --ignore-missing --quiet --check checksums.txt ||
+            ( echo  "Error while verifying checksums !" >&2 && exit 1 )
+    )
+
+    # Extracting archive files
+    tar -xf "$TAR_FILE" -C "$FORMIDABLE_TMPDIR"
+
+    # Moving downloaded binary to destination directory
+    mv -f "$FORMIDABLE_TMPDIR/$FORMIDABLE_FILE_BASENAME" "$FORMIDABLE_DESTDIR/"
+}
+
+main $@