feat: url based data loading system

internal/command/common.go

@@ -1,11 +1,17 @@
 package command
 
 import (
-	"encoding/json"
+	"bytes"
 	"io"
+	"net/url"
 	"os"
-	"strings"
 
+	encjson "encoding/json"
+
+	"forge.cadoles.com/wpetit/formidable/internal/data"
+	"forge.cadoles.com/wpetit/formidable/internal/data/format/hcl"
+	"forge.cadoles.com/wpetit/formidable/internal/data/format/json"
+	"forge.cadoles.com/wpetit/formidable/internal/data/scheme/file"
 	"forge.cadoles.com/wpetit/formidable/internal/def"
 	"github.com/pkg/errors"
 	"github.com/santhosh-tekuri/jsonschema/v5"
@@ -22,13 +28,13 @@ func commonFlags() []cli.Flag {
 			Name:    "defaults",
 			Aliases: []string{"d"},
 			Usage:   "Default values as JSON or file path prefixed by '@'",
-			Value:   "{}",
+			Value:   "",
 		},
 		&cli.StringFlag{
 			Name:    "values",
 			Aliases: []string{"v"},
 			Usage:   "Current values as JSON or file path prefixed by '@'",
-			Value:   "{}",
+			Value:   "",
 		},
 		&cli.StringFlag{
 			Name:      "schema",
@@ -45,49 +51,43 @@ func commonFlags() []cli.Flag {
 	}
 }
 
-func loadJSONFlag(ctx *cli.Context, flagName string) (interface{}, error) {
+func loadURLFlag(ctx *cli.Context, flagName string) (interface{}, error) {
 	flagValue := ctx.String(flagName)
 
 	if flagValue == "" {
 		return nil, nil
 	}
 
-	if !strings.HasPrefix(flagValue, filePathPrefix) {
-		var value interface{}
+	loader := newLoader()
 
-		if err := json.Unmarshal([]byte(flagValue), &value); err != nil {
-			return nil, errors.WithStack(err)
-		}
-
-		return value, nil
+	url, err := url.Parse(flagValue)
+	if err != nil {
+		return nil, errors.WithStack(err)
 	}
 
-	flagValue = strings.TrimPrefix(flagValue, filePathPrefix)
-
-	file, err := os.Open(flagValue)
+	reader, err := loader.Open(url)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
 	defer func() {
-		if err := file.Close(); err != nil {
+		if err := reader.Close(); err != nil {
 			panic(errors.WithStack(err))
 		}
 	}()
 
-	reader := json.NewDecoder(file)
+	decoder := newDecoder()
 
-	var values interface{}
-
-	if err := reader.Decode(&values); err != nil {
+	data, err := decoder.Decode(url, reader)
+	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
-	return values, nil
+	return data, nil
 }
 
 func loadValues(ctx *cli.Context) (interface{}, error) {
-	values, err := loadJSONFlag(ctx, "values")
+	values, err := loadURLFlag(ctx, "values")
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -96,33 +96,45 @@ func loadValues(ctx *cli.Context) (interface{}, error) {
 }
 
 func loadDefaults(ctx *cli.Context) (interface{}, error) {
-	values, err := loadJSONFlag(ctx, "defaults")
+	defaults, err := loadURLFlag(ctx, "defaults")
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
-	return values, nil
+	return defaults, nil
 }
 
 func loadSchema(ctx *cli.Context) (*jsonschema.Schema, error) {
 	schemaFlag := ctx.String("schema")
 
+	if schemaFlag == "" {
+		return def.Schema, nil
+	}
+
+	schemaTree, err := loadURLFlag(ctx, "schema")
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	// Reencode schema to JSON format
+	var buf bytes.Buffer
+	encoder := encjson.NewEncoder(&buf)
+
+	if err := encoder.Encode(schemaTree); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
 	compiler := jsonschema.NewCompiler()
 
 	compiler.ExtractAnnotations = true
 	compiler.AssertFormat = true
 	compiler.AssertContent = true
 
-	var (
-		schema *jsonschema.Schema
-		err    error
-	)
-
-	if schemaFlag == "" {
-		schema = def.Schema
-	} else {
-		schema, err = compiler.Compile(schemaFlag)
+	if err := compiler.AddResource(schemaFlag, &buf); err != nil {
+		return nil, errors.WithStack(err)
 	}
+
+	schema, err := compiler.Compile(schemaFlag)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
@@ -154,3 +166,16 @@ func outputWriter(ctx *cli.Context) (io.WriteCloser, error) {
 
 	return file, nil
 }
+
+func newLoader() *data.Loader {
+	return data.NewLoader(
+		file.NewLoaderHandler(),
+	)
+}
+
+func newDecoder() *data.Decoder {
+	return data.NewDecoder(
+		json.NewDecoderHandler(),
+		hcl.NewDecoderHandler(nil),
+	)
+}

internal/data/decoder.go

@@ -0,0 +1,40 @@
+package data
+
+import (
+	"io"
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+const FormatQueryParam = "format"
+
+type DecoderHandler interface {
+	URLMatcher
+	Decode(url *url.URL, reader io.Reader) (interface{}, error)
+}
+
+type Decoder struct {
+	handlers []DecoderHandler
+}
+
+func (d *Decoder) Decode(url *url.URL, reader io.ReadCloser) (interface{}, error) {
+	for _, h := range d.handlers {
+		if !h.Match(url) {
+			continue
+		}
+
+		data, err := h.Decode(url, reader)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return data, nil
+	}
+
+	return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String())
+}
+
+func NewDecoder(handlers ...DecoderHandler) *Decoder {
+	return &Decoder{handlers}
+}

internal/data/error.go

@@ -0,0 +1,5 @@
+package data
+
+import "github.com/pkg/errors"
+
+var ErrHandlerNotFound = errors.New("handler not found")

internal/data/format/hcl/decoder_handler.go

@@ -0,0 +1,165 @@
+package hcl
+
+import (
+	"io"
+	"net/url"
+	"path"
+	"path/filepath"
+
+	"forge.cadoles.com/wpetit/formidable/internal/data/format"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/gohcl"
+	"github.com/hashicorp/hcl/v2/hclparse"
+	"github.com/pkg/errors"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/gocty"
+)
+
+const (
+	ExtensionHCL = ".hcl"
+	FormatHCL    = "hcl"
+)
+
+type DecoderHandler struct {
+	ctx *hcl.EvalContext
+}
+
+func (d *DecoderHandler) Match(url *url.URL) bool {
+	ext := filepath.Ext(path.Join(url.Host, url.Path))
+
+	return ext == ExtensionHCL ||
+		format.MatchURLQueryFormat(url, FormatHCL)
+}
+
+func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	ctx := d.ctx
+	if ctx == nil {
+		ctx = &hcl.EvalContext{
+			Variables: make(map[string]cty.Value),
+			Functions: make(map[string]function.Function),
+		}
+	}
+
+	parser := hclparse.NewParser()
+
+	file, diags := parser.ParseHCL(data, url.String())
+	if diags.HasErrors() {
+		return nil, errors.WithStack(diags)
+	}
+
+	var tree map[string]interface{}
+
+	diags = gohcl.DecodeBody(file.Body, ctx, &tree)
+	if diags.HasErrors() {
+		return nil, errors.WithStack(diags)
+	}
+
+	ctx = ctx.NewChild()
+
+	values, err := hclTreeToRawValues(ctx, tree)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return values, nil
+}
+
+func NewDecoderHandler(ctx *hcl.EvalContext) *DecoderHandler {
+	return &DecoderHandler{ctx}
+}
+
+func hclTreeToRawValues(ctx *hcl.EvalContext, tree map[string]interface{}) (map[string]interface{}, error) {
+	values := make(map[string]interface{})
+
+	for key, branch := range tree {
+		v, err := hclBranchToRawValue(ctx, branch)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		values[key] = v
+	}
+
+	return values, nil
+}
+
+func hclBranchToRawValue(ctx *hcl.EvalContext, branch interface{}) (interface{}, error) {
+	switch typ := branch.(type) {
+	case *hcl.Attribute:
+		val, diags := typ.Expr.Value(ctx)
+		if diags.HasErrors() {
+			return nil, errors.WithStack(diags)
+		}
+
+		raw, err := ctyValueToRaw(ctx, val)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return raw, nil
+	default:
+		return nil, errors.Errorf("unexpected type '%T'", typ)
+	}
+}
+
+func ctyValueToRaw(ctx *hcl.EvalContext, val cty.Value) (interface{}, error) {
+	if val.Type().Equals(cty.Bool) {
+		var raw bool
+
+		if err := gocty.FromCtyValue(val, &raw); err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return raw, nil
+	} else if val.Type().Equals(cty.Number) {
+		var raw float64
+
+		if err := gocty.FromCtyValue(val, &raw); err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return raw, nil
+	} else if val.Type().Equals(cty.String) {
+		var raw string
+
+		if err := gocty.FromCtyValue(val, &raw); err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return raw, nil
+	} else if val.Type().IsObjectType() {
+		obj := make(map[string]interface{})
+
+		for k, v := range val.AsValueMap() {
+			rv, err := ctyValueToRaw(ctx, v)
+			if err != nil {
+				return nil, errors.WithStack(err)
+			}
+
+			obj[k] = rv
+		}
+
+		return obj, nil
+	} else if val.Type().IsTupleType() {
+		sl := make([]interface{}, 0)
+
+		for _, v := range val.AsValueSlice() {
+			rv, err := ctyValueToRaw(ctx, v)
+			if err != nil {
+				return nil, errors.WithStack(err)
+			}
+
+			sl = append(sl, rv)
+		}
+
+		return sl, nil
+	}
+
+	return nil, errors.Errorf("unexpected cty.Type '%s'", val.Type().FriendlyName())
+}

internal/data/format/hcl/decoder_handler_test.go

@@ -0,0 +1,78 @@
+package hcl
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+type parserHandlerTestCase struct {
+	Path             string
+	ExpectMatch      bool
+	ExpectParseError bool
+}
+
+var parserHandlerTestCases = []parserHandlerTestCase{
+	{
+		Path:             "testdata/dummy.hcl",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+	{
+		Path:             "file://testdata/dummy_no_ext?format=hcl",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+}
+
+func TestDecoderHandler(t *testing.T) {
+	t.Parallel()
+
+	handler := NewDecoderHandler(nil)
+
+	for _, tc := range parserHandlerTestCases {
+		func(tc parserHandlerTestCase) {
+			t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) {
+				t.Parallel()
+
+				url, err := url.Parse(tc.Path)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path))
+				}
+
+				if e, g := tc.ExpectMatch, handler.Match(url); e != g {
+					t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g)
+				}
+
+				if !tc.ExpectMatch {
+					return
+				}
+
+				cleanedPath := filepath.Join(url.Host, url.Path)
+
+				file, err := os.Open(cleanedPath)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath))
+				}
+
+				defer func() {
+					if err := file.Close(); err != nil {
+						t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath))
+					}
+				}()
+
+				if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError {
+					t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path))
+				}
+
+				if tc.ExpectParseError {
+					t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
+				}
+			})
+		}(tc)
+	}
+}

internal/data/format/hcl/testdata/dummy.hcl

@@ -0,0 +1,19 @@
+test = 1
+
+test1 = 2 + 1
+
+foo = {
+  description = "Ça fait des trucs"
+  type = "object"
+  properties = {
+    type = "string"
+    minLength = 5
+  }
+  test = [
+    "foo", 
+    {
+      test = "foo"
+    },
+    5 + 5.2
+  ]
+}

internal/data/format/hcl/testdata/dummy_no_ext

@@ -0,0 +1 @@
+foo = "bar"

internal/data/format/json/decoder_handler.go

@@ -0,0 +1,42 @@
+package json
+
+import (
+	"encoding/json"
+	"io"
+	"net/url"
+	"path"
+	"path/filepath"
+
+	"forge.cadoles.com/wpetit/formidable/internal/data/format"
+	"github.com/pkg/errors"
+)
+
+const (
+	ExtensionJSON = ".json"
+	FormatJSON    = "json"
+)
+
+type DecoderHandler struct{}
+
+func (d *DecoderHandler) Match(url *url.URL) bool {
+	ext := filepath.Ext(path.Join(url.Host, url.Path))
+
+	return ext == ExtensionJSON ||
+		format.MatchURLQueryFormat(url, FormatJSON)
+}
+
+func (d *DecoderHandler) Decode(url *url.URL, reader io.Reader) (interface{}, error) {
+	decoder := json.NewDecoder(reader)
+
+	var values interface{}
+
+	if err := decoder.Decode(&values); err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	return values, nil
+}
+
+func NewDecoderHandler() *DecoderHandler {
+	return &DecoderHandler{}
+}

internal/data/format/json/decoder_handler_test.go

@@ -0,0 +1,78 @@
+package json
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+type parserHandlerTestCase struct {
+	Path             string
+	ExpectMatch      bool
+	ExpectParseError bool
+}
+
+var parserHandlerTestCases = []parserHandlerTestCase{
+	{
+		Path:             "testdata/dummy.json",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+	{
+		Path:             "file://testdata/dummy_no_ext?format=json",
+		ExpectMatch:      true,
+		ExpectParseError: false,
+	},
+}
+
+func TestDecoderHandler(t *testing.T) {
+	t.Parallel()
+
+	handler := NewDecoderHandler()
+
+	for _, tc := range parserHandlerTestCases {
+		func(tc parserHandlerTestCase) {
+			t.Run(fmt.Sprintf("Parse '%s'", tc.Path), func(t *testing.T) {
+				t.Parallel()
+
+				url, err := url.Parse(tc.Path)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.Path))
+				}
+
+				if e, g := tc.ExpectMatch, handler.Match(url); e != g {
+					t.Errorf("URL '%s': expected matching result '%v', got '%v'", url.String(), e, g)
+				}
+
+				if !tc.ExpectMatch {
+					return
+				}
+
+				cleanedPath := filepath.Join(url.Host, url.Path)
+
+				file, err := os.Open(cleanedPath)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not open file '%s'", cleanedPath))
+				}
+
+				defer func() {
+					if err := file.Close(); err != nil {
+						t.Error(errors.Wrapf(err, "could not close file '%s'", cleanedPath))
+					}
+				}()
+
+				if _, err := handler.Decode(url, file); err != nil && !tc.ExpectParseError {
+					t.Fatal(errors.Wrapf(err, "could not parse file '%s'", tc.Path))
+				}
+
+				if tc.ExpectParseError {
+					t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
+				}
+			})
+		}(tc)
+	}
+}

internal/data/format/json/testdata/dummy.json

@@ -0,0 +1,3 @@
+{
+    "foo": "bar"
+}

internal/data/format/json/testdata/dummy_no_ext

@@ -0,0 +1 @@
+{}

internal/data/format/query.go

@@ -0,0 +1,7 @@
+package format
+
+import "net/url"
+
+func MatchURLQueryFormat(url *url.URL, format string) bool {
+	return url.Query().Get("format") == format
+}

internal/data/loader.go

@@ -0,0 +1,38 @@
+package data
+
+import (
+	"io"
+	"net/url"
+
+	"github.com/pkg/errors"
+)
+
+type LoaderHandler interface {
+	URLMatcher
+	Open(url *url.URL) (io.ReadCloser, error)
+}
+
+type Loader struct {
+	handlers []LoaderHandler
+}
+
+func (l *Loader) Open(url *url.URL) (io.ReadCloser, error) {
+	for _, h := range l.handlers {
+		if !h.Match(url) {
+			continue
+		}
+
+		reader, err := h.Open(url)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+
+		return reader, nil
+	}
+
+	return nil, errors.Wrapf(ErrHandlerNotFound, "could not find matching handler for url '%s'", url.String())
+}
+
+func NewLoader(handlers ...LoaderHandler) *Loader {
+	return &Loader{handlers}
+}

internal/data/scheme/file/loader_handler.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+const SchemeFile = "file"
+
+type LoaderHandler struct{}
+
+func (h *LoaderHandler) Match(url *url.URL) bool {
+	return url.Scheme == SchemeFile || url.Scheme == ""
+}
+
+func (h *LoaderHandler) Open(url *url.URL) (io.ReadCloser, error) {
+	file, err := os.Open(filepath.Join(url.Host, url.Path))
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not open file '%s'", url.Path)
+	}
+
+	return file, nil
+}
+
+func NewLoaderHandler() *LoaderHandler {
+	return &LoaderHandler{}
+}

internal/data/scheme/file/loader_handler_test.go

@@ -0,0 +1,107 @@
+package file
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/errors"
+)
+
+const dummyFilePath = "testdata/dummy.txt"
+
+var loaderHandlerTestCases []loaderHandlerTestCase
+
+func init() {
+	dummyFileAbsolutePath, err := filepath.Abs(dummyFilePath)
+	if err != nil {
+		panic(errors.WithStack(err))
+	}
+
+	loaderHandlerTestCases = []loaderHandlerTestCase{
+		{
+			URL:               SchemeFile + "://" + dummyFilePath,
+			ExpectMatch:       true,
+			ExpectOpenError:   false,
+			ExpectOpenContent: "dummy",
+		},
+		{
+			URL:               dummyFilePath,
+			ExpectMatch:       true,
+			ExpectOpenError:   false,
+			ExpectOpenContent: "dummy",
+		},
+		{
+			URL:               dummyFileAbsolutePath,
+			ExpectMatch:       true,
+			ExpectOpenError:   false,
+			ExpectOpenContent: "dummy",
+		},
+		{
+			URL:               SchemeFile + "://" + dummyFileAbsolutePath,
+			ExpectMatch:       true,
+			ExpectOpenError:   false,
+			ExpectOpenContent: "dummy",
+		},
+	}
+}
+
+type loaderHandlerTestCase struct {
+	URL               string
+	ExpectMatch       bool
+	ExpectOpenError   bool
+	ExpectOpenContent string
+}
+
+func TestLoaderHandler(t *testing.T) {
+	t.Parallel()
+
+	handler := NewLoaderHandler()
+
+	for _, tc := range loaderHandlerTestCases {
+		func(tc loaderHandlerTestCase) {
+			t.Run(fmt.Sprintf("Load '%s'", tc.URL), func(t *testing.T) {
+				t.Parallel()
+
+				url, err := url.Parse(tc.URL)
+				if err != nil {
+					t.Fatal(errors.Wrapf(err, "could not parse url '%s'", tc.URL))
+				}
+
+				if e, g := tc.ExpectMatch, handler.Match(url); e != g {
+					t.Errorf("URL '%s': expected matching result '%v', got '%v'", tc.URL, e, g)
+				}
+
+				if !tc.ExpectMatch {
+					return
+				}
+
+				reader, err := handler.Open(url)
+				if err != nil && !tc.ExpectOpenError {
+					t.Fatal(errors.Wrapf(err, "could not open url '%s'", url.String()))
+				}
+
+				defer func() {
+					if err := reader.Close(); err != nil {
+						t.Error(errors.WithStack(err))
+					}
+				}()
+
+				if tc.ExpectOpenError {
+					t.Fatal(errors.Errorf("no error was returned as expected when opening url '%s'", url.String()))
+				}
+
+				data, err := io.ReadAll(reader)
+				if err != nil {
+					t.Fatal(errors.WithStack(err))
+				}
+
+				if e, g := tc.ExpectOpenContent, string(data); e != g {
+					t.Errorf("URL '%s': expected content'%v', got '%v'", tc.URL, e, g)
+				}
+			})
+		}(tc)
+	}
+}

internal/data/scheme/file/testdata/dummy.txt

@@ -0,0 +1 @@
+dummy

internal/data/url_matcher.go

@@ -0,0 +1,7 @@
+package data
+
+import "net/url"
+
+type URLMatcher interface {
+	Match(url *url.URL) bool
+}