feat(storage): rpc based implementation
All checks were successful
arcad/edge/pipeline/pr-master This commit looks good

This commit is contained in:
2023-09-12 22:03:25 -06:00
parent c3535a4a9b
commit 8e574c299b
113 changed files with 3007 additions and 263 deletions

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
var blobStoreFactories = make(map[string]BlobStoreFactory, 0)
type BlobStoreFactory func(url *url.URL) (storage.BlobStore, error)
func RegisterBlobStoreFactory(scheme string, factory BlobStoreFactory) {
blobStoreFactories[scheme] = factory
}
func NewBlobStore(dsn string) (storage.BlobStore, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := blobStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
var documentStoreFactories = make(map[string]DocumentStoreFactory, 0)
type DocumentStoreFactory func(url *url.URL) (storage.DocumentStore, error)
func RegisterDocumentStoreFactory(scheme string, factory DocumentStoreFactory) {
documentStoreFactories[scheme] = factory
}
func NewDocumentStore(dsn string) (storage.DocumentStore, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := documentStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -0,0 +1,5 @@
package driver
import "errors"
var ErrSchemeNotRegistered = errors.New("scheme was not registered")

View File

@ -0,0 +1,239 @@
package client
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"github.com/pkg/errors"
)
type BlobBucket struct {
name string
id blob.BucketID
call CallFunc
}
// Size implements storage.BlobBucket
func (b *BlobBucket) Size(ctx context.Context) (int64, error) {
args := blob.GetBucketSizeArgs{
BucketID: b.id,
}
reply := blob.GetBucketSizeReply{}
if err := b.call(ctx, "Service.GetBucketSize", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Size, nil
}
// Name implements storage.BlobBucket
func (b *BlobBucket) Name() string {
return b.name
}
// Close implements storage.BlobBucket
func (b *BlobBucket) Close() error {
args := blob.CloseBucketArgs{
BucketID: b.id,
}
reply := blob.CloseBucketReply{}
if err := b.call(context.Background(), "Service.CloseBucket", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Delete implements storage.BlobBucket
func (b *BlobBucket) Delete(ctx context.Context, id storage.BlobID) error {
args := blob.DeleteBucketArgs{
BucketName: b.name,
}
reply := blob.DeleteBucketReply{}
if err := b.call(context.Background(), "Service.DeleteBucket", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.BlobBucket
func (b *BlobBucket) Get(ctx context.Context, id storage.BlobID) (storage.BlobInfo, error) {
args := blob.GetBlobInfoArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.GetBlobInfoReply{}
if err := b.call(context.Background(), "Service.GetBlobInfo", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.BlobInfo, nil
}
// List implements storage.BlobBucket
func (b *BlobBucket) List(ctx context.Context) ([]storage.BlobInfo, error) {
args := blob.ListBlobInfoArgs{
BucketID: b.id,
}
reply := blob.ListBlobInfoReply{}
if err := b.call(context.Background(), "Service.ListBlobInfo", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.BlobInfos, nil
}
// NewReader implements storage.BlobBucket
func (b *BlobBucket) NewReader(ctx context.Context, id storage.BlobID) (io.ReadSeekCloser, error) {
args := blob.NewBlobReaderArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.NewBlobReaderReply{}
if err := b.call(context.Background(), "Service.NewBlobReader", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return &blobReaderCloser{
readerID: reply.ReaderID,
call: b.call,
}, nil
}
// NewWriter implements storage.BlobBucket
func (b *BlobBucket) NewWriter(ctx context.Context, id storage.BlobID) (io.WriteCloser, error) {
args := blob.NewBlobWriterArgs{
BucketID: b.id,
BlobID: id,
}
reply := blob.NewBlobWriterReply{}
if err := b.call(context.Background(), "Service.NewBlobWriter", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return &blobWriterCloser{
blobID: id,
writerID: reply.WriterID,
call: b.call,
}, nil
}
type blobWriterCloser struct {
blobID storage.BlobID
writerID blob.WriterID
call CallFunc
}
// Write implements io.WriteCloser
func (bwc *blobWriterCloser) Write(data []byte) (int, error) {
args := blob.WriteBlobArgs{
WriterID: bwc.writerID,
Data: data,
}
reply := blob.WriteBlobReply{}
if err := bwc.call(context.Background(), "Service.WriteBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Written, nil
}
// Close implements io.WriteCloser
func (bwc *blobWriterCloser) Close() error {
args := blob.CloseWriterArgs{
WriterID: bwc.writerID,
}
reply := blob.CloseBucketReply{}
if err := bwc.call(context.Background(), "Service.CloseWriter", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
type blobReaderCloser struct {
readerID blob.ReaderID
call func(ctx context.Context, serviceMethod string, args any, reply any) error
}
// Read implements io.ReadSeekCloser
func (brc *blobReaderCloser) Read(p []byte) (int, error) {
args := blob.ReadBlobArgs{
ReaderID: brc.readerID,
Length: len(p),
}
reply := blob.ReadBlobReply{}
if err := brc.call(context.Background(), "Service.ReadBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
copy(p, reply.Data)
if reply.EOF {
return reply.Read, io.EOF
}
return reply.Read, nil
}
// Seek implements io.ReadSeekCloser
func (brc *blobReaderCloser) Seek(offset int64, whence int) (int64, error) {
args := blob.SeekBlobArgs{
ReaderID: brc.readerID,
Offset: offset,
Whence: whence,
}
reply := blob.SeekBlobReply{}
if err := brc.call(context.Background(), "Service.SeekBlob", args, &reply); err != nil {
return 0, errors.WithStack(err)
}
return reply.Read, nil
}
// Close implements io.ReadSeekCloser
func (brc *blobReaderCloser) Close() error {
args := blob.CloseReaderArgs{
ReaderID: brc.readerID,
}
reply := blob.CloseReaderReply{}
if err := brc.call(context.Background(), "Service.CloseReader", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
var (
_ storage.BlobBucket = &BlobBucket{}
_ storage.BlobInfo = &BlobInfo{}
_ io.WriteCloser = &blobWriterCloser{}
_ io.ReadSeekCloser = &blobReaderCloser{}
)

View File

@ -0,0 +1,40 @@
package client
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
type BlobInfo struct {
id storage.BlobID
bucket string
contentType string
modTime time.Time
size int64
}
// Bucket implements storage.BlobInfo
func (i *BlobInfo) Bucket() string {
return i.bucket
}
// ID implements storage.BlobInfo
func (i *BlobInfo) ID() storage.BlobID {
return i.id
}
// ContentType implements storage.BlobInfo
func (i *BlobInfo) ContentType() string {
return i.contentType
}
// ModTime implements storage.BlobInfo
func (i *BlobInfo) ModTime() time.Time {
return i.modTime
}
// Size implements storage.BlobInfo
func (i *BlobInfo) Size() int64 {
return i.size
}

View File

@ -0,0 +1,101 @@
package client
import (
"context"
"net/url"
"github.com/keegancsmith/rpc"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"github.com/pkg/errors"
)
type BlobStore struct {
serverURL *url.URL
}
// DeleteBucket implements storage.BlobStore.
func (s *BlobStore) DeleteBucket(ctx context.Context, name string) error {
args := &blob.DeleteBucketArgs{
BucketName: name,
}
if err := s.call(ctx, "Service.DeleteBucket", args, nil); err != nil {
return errors.WithStack(err)
}
return nil
}
// ListBuckets implements storage.BlobStore.
func (s *BlobStore) ListBuckets(ctx context.Context) ([]string, error) {
args := &blob.ListBucketsArgs{}
reply := blob.ListBucketsReply{}
if err := s.call(ctx, "Service.ListBuckets", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Buckets, nil
}
// OpenBucket implements storage.BlobStore.
func (s *BlobStore) OpenBucket(ctx context.Context, name string) (storage.BlobBucket, error) {
args := &blob.OpenBucketArgs{
BucketName: name,
}
reply := &blob.OpenBucketReply{}
if err := s.call(ctx, "Service.OpenBucket", args, reply); err != nil {
return nil, errors.WithStack(err)
}
return &BlobBucket{
name: name,
id: reply.BucketID,
call: s.call,
}, nil
}
func (s *BlobStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *BlobStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewBlobStore(serverURL *url.URL) *BlobStore {
return &BlobStore{serverURL}
}
var _ storage.BlobStore = &BlobStore{}

View File

@ -0,0 +1,87 @@
package client
import (
"context"
"fmt"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestBlobStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
httpServer, err := startNewBlobStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewBlobStore(serverURL)
testsuite.TestBlobStore(context.Background(), t, store)
}
func BenchmarkBlobStore(t *testing.B) {
logger.SetLevel(logger.LevelError)
httpServer, err := startNewBlobStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewBlobStore(serverURL)
testsuite.BenchmarkBlobStore(t, store)
}
func getSQLiteBlobStore() (*sqlite.BlobStore, error) {
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewBlobStore(dsn)
return store, nil
}
func startNewBlobStoreServer() (*httptest.Server, error) {
store, err := getSQLiteBlobStore()
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewBlobStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1,134 @@
package client
import (
"context"
"net/url"
"github.com/keegancsmith/rpc"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/document"
"forge.cadoles.com/arcad/edge/pkg/storage/filter"
)
type DocumentStore struct {
serverURL *url.URL
}
// Delete implements storage.DocumentStore.
func (s *DocumentStore) Delete(ctx context.Context, collection string, id storage.DocumentID) error {
args := document.DeleteDocumentArgs{
Collection: collection,
DocumentID: id,
}
reply := document.DeleteDocumentReply{}
if err := s.call(ctx, "Service.DeleteDocument", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.DocumentStore.
func (s *DocumentStore) Get(ctx context.Context, collection string, id storage.DocumentID) (storage.Document, error) {
args := document.GetDocumentArgs{
Collection: collection,
DocumentID: id,
}
reply := document.GetDocumentReply{}
if err := s.call(ctx, "Service.GetDocument", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Document, nil
}
// Query implements storage.DocumentStore.
func (s *DocumentStore) Query(ctx context.Context, collection string, filter *filter.Filter, funcs ...storage.QueryOptionFunc) ([]storage.Document, error) {
opts := &storage.QueryOptions{}
for _, fn := range funcs {
fn(opts)
}
args := document.QueryDocumentsArgs{
Collection: collection,
Filter: nil,
Options: opts,
}
if filter != nil {
args.Filter = filter.AsMap()
}
reply := document.QueryDocumentsReply{
Documents: []storage.Document{},
}
if err := s.call(ctx, "Service.QueryDocuments", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Documents, nil
}
// Upsert implements storage.DocumentStore.
func (s *DocumentStore) Upsert(ctx context.Context, collection string, doc storage.Document) (storage.Document, error) {
args := document.UpsertDocumentArgs{
Collection: collection,
Document: doc,
}
reply := document.UpsertDocumentReply{}
if err := s.call(ctx, "Service.UpsertDocument", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Document, nil
}
func (s *DocumentStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *DocumentStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewDocumentStore(url *url.URL) *DocumentStore {
return &DocumentStore{url}
}
var _ storage.DocumentStore = &DocumentStore{}

View File

@ -0,0 +1,67 @@
package client
import (
"context"
"fmt"
"net/http/httptest"
"net/url"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestDocumentStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
httpServer, err := startNewDocumentStoreServer()
if err != nil {
t.Fatalf("%+v", errors.WithStack(err))
}
defer httpServer.Close()
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
store := NewDocumentStore(serverURL)
testsuite.TestDocumentStore(context.Background(), t, store)
}
func getSQLiteDocumentStore() (*sqlite.DocumentStore, error) {
file := "./testdata/documentstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewDocumentStore(dsn)
return store, nil
}
func startNewDocumentStoreServer() (*httptest.Server, error) {
store, err := getSQLiteDocumentStore()
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewDocumentStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1,17 @@
package client
import (
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
func remapShareError(err error) error {
switch errors.Cause(err).Error() {
case share.ErrAttributeRequired.Error():
return share.ErrAttributeRequired
case share.ErrNotFound.Error():
return share.ErrNotFound
default:
return err
}
}

View File

@ -0,0 +1,9 @@
package client
import (
"context"
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
)
type CallFunc func(ctx context.Context, serviceMethod string, args any, reply any) error

View File

@ -0,0 +1,150 @@
package client
import (
"context"
"net/url"
"forge.cadoles.com/arcad/edge/pkg/app"
server "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/keegancsmith/rpc"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type ShareStore struct {
serverURL *url.URL
}
// DeleteAttributes implements share.Store.
func (s *ShareStore) DeleteAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, names ...string) error {
args := server.DeleteAttributesArgs{
Origin: origin,
ResourceID: resourceID,
Names: names,
}
reply := server.DeleteAttributesArgs{}
if err := s.call(ctx, "Service.DeleteAttributes", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// DeleteResource implements share.Store.
func (s *ShareStore) DeleteResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) error {
args := server.DeleteResourceArgs{
Origin: origin,
ResourceID: resourceID,
}
reply := server.DeleteResourceReply{}
if err := s.call(ctx, "Service.DeleteResource", args, &reply); err != nil {
return errors.WithStack(err)
}
return nil
}
// FindResources implements share.Store.
func (s *ShareStore) FindResources(ctx context.Context, funcs ...share.FindResourcesOptionFunc) ([]share.Resource, error) {
options := share.NewFindResourcesOptions(funcs...)
args := server.FindResourcesArgs{
Options: options,
}
reply := server.FindResourcesReply{}
if err := s.call(ctx, "Service.FindResources", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
resources := make([]share.Resource, len(reply.Resources))
for idx, res := range reply.Resources {
resources[idx] = res
}
return resources, nil
}
// GetResource implements share.Store.
func (s *ShareStore) GetResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) (share.Resource, error) {
args := server.GetResourceArgs{
Origin: origin,
ResourceID: resourceID,
}
reply := server.GetResourceReply{}
if err := s.call(ctx, "Service.GetResource", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Resource, nil
}
// UpdateAttributes implements share.Store.
func (s *ShareStore) UpdateAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, attributes ...share.Attribute) (share.Resource, error) {
serializableAttributes := make([]*server.SerializableAttribute, len(attributes))
for attrIdx, attr := range attributes {
serializableAttributes[attrIdx] = server.FromAttribute(attr)
}
args := server.UpdateAttributesArgs{
Origin: origin,
ResourceID: resourceID,
Attributes: serializableAttributes,
}
reply := server.UpdateAttributesReply{}
if err := s.call(ctx, "Service.UpdateAttributes", args, &reply); err != nil {
return nil, errors.WithStack(err)
}
return reply.Resource, nil
}
func (s *ShareStore) call(ctx context.Context, serviceMethod string, args any, reply any) error {
err := s.withClient(ctx, func(ctx context.Context, client *rpc.Client) error {
if err := client.Call(ctx, serviceMethod, args, reply); err != nil {
return errors.WithStack(remapShareError(err))
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *ShareStore) withClient(ctx context.Context, fn func(ctx context.Context, client *rpc.Client) error) error {
client, err := rpc.DialHTTPPath("tcp", s.serverURL.Host, s.serverURL.Path+"?"+s.serverURL.RawQuery)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := client.Close(); err != nil {
logger.Error(ctx, "could not close rpc client", logger.E(errors.WithStack(err)))
}
}()
if err := fn(ctx, client); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewShareStore(url *url.URL) *ShareStore {
return &ShareStore{url}
}
var _ share.Store = &ShareStore{}

View File

@ -0,0 +1,67 @@
package client
import (
"fmt"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestShareStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
testsuite.TestStore(t, func(testName string) (share.Store, error) {
httpServer, err := startNewShareStoreServer(testName)
if err != nil {
return nil, errors.WithStack(err)
}
serverAddr := httpServer.Listener.Addr()
serverURL := &url.URL{
Host: serverAddr.String(),
}
return NewShareStore(serverURL), nil
})
}
func getSQLiteShareStore(testName string) (*sqlite.ShareStore, error) {
filename := strings.ToLower(strings.ReplaceAll(testName, " ", "_"))
file := fmt.Sprintf("./testdata/sharestore_test_%s.sqlite", filename)
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := sqlite.NewShareStore(dsn)
return store, nil
}
func startNewShareStoreServer(testName string) (*httptest.Server, error) {
store, err := getSQLiteShareStore(testName)
if err != nil {
return nil, errors.WithStack(err)
}
server := server.NewShareStoreServer(store)
httpServer := httptest.NewServer(server)
return httpServer, nil
}

View File

@ -0,0 +1 @@
/*.sqlite*

View File

@ -0,0 +1,22 @@
package rpc
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/client"
)
func init() {
driver.RegisterDocumentStoreFactory("rpc", documentStoreFactory)
driver.RegisterBlobStoreFactory("rpc", blobStoreFactory)
}
func documentStoreFactory(url *url.URL) (storage.DocumentStore, error) {
return client.NewDocumentStore(url), nil
}
func blobStoreFactory(url *url.URL) (storage.BlobStore, error) {
return client.NewBlobStore(url), nil
}

View File

@ -0,0 +1,42 @@
package gob
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
type BlobInfo struct {
Bucket_ string
ContentType_ string
BlobID_ storage.BlobID
ModTime_ time.Time
Size_ int64
}
// Bucket implements storage.BlobInfo.
func (bi *BlobInfo) Bucket() string {
return bi.Bucket_
}
// ContentType implements storage.BlobInfo.
func (bi *BlobInfo) ContentType() string {
return bi.ContentType_
}
// ID implements storage.BlobInfo.
func (bi *BlobInfo) ID() storage.BlobID {
return bi.BlobID_
}
// ModTime implements storage.BlobInfo.
func (bi *BlobInfo) ModTime() time.Time {
return bi.ModTime_
}
// Size implements storage.BlobInfo.
func (bi *BlobInfo) Size() int64 {
return bi.Size_
}
var _ storage.BlobInfo = &BlobInfo{}

View File

@ -0,0 +1,18 @@
package gob
import (
"encoding/gob"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
func init() {
gob.Register(storage.Document{})
gob.Register(storage.DocumentID(""))
gob.Register(time.Time{})
gob.Register(map[string]interface{}{})
gob.Register([]interface{}{})
gob.Register([]map[string]interface{}{})
gob.Register(&BlobInfo{})
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseBucketArgs struct {
BucketID BucketID
}
type CloseBucketReply struct {
}
func (s *Service) CloseBucket(ctx context.Context, args *CloseBucketArgs, reply *CloseBucketReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
if err := bucket.Close(); err != nil {
return errors.WithStack(err)
}
s.buckets.Delete(args.BucketID)
*reply = CloseBucketReply{}
return nil
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseReaderArgs struct {
ReaderID ReaderID
}
type CloseReaderReply struct {
}
func (s *Service) CloseReader(ctx context.Context, args *CloseReaderArgs, reply *CloseReaderReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
if err := reader.Close(); err != nil {
return errors.WithStack(err)
}
s.readers.Delete(args.ReaderID)
*reply = CloseReaderReply{}
return nil
}

View File

@ -0,0 +1,31 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type CloseWriterArgs struct {
WriterID WriterID
}
type CloseWriterReply struct {
}
func (s *Service) CloseWriter(ctx context.Context, args *CloseWriterArgs, reply *CloseWriterReply) error {
writer, err := s.getOpenedWriter(args.WriterID)
if err != nil {
return errors.WithStack(err)
}
if err := writer.Close(); err != nil {
return errors.WithStack(err)
}
s.writers.Delete(args.WriterID)
*reply = CloseWriterReply{}
return nil
}

View File

@ -0,0 +1,22 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type DeleteBucketArgs struct {
BucketName string
}
type DeleteBucketReply struct {
}
func (s *Service) DeleteBucket(ctx context.Context, args *DeleteBucketArgs, reply *DeleteBucketReply) error {
if err := s.store.DeleteBucket(ctx, args.BucketName); err != nil {
return errors.WithStack(err)
}
return nil
}

View File

@ -0,0 +1,42 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
"github.com/pkg/errors"
)
type GetBlobInfoArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type GetBlobInfoReply struct {
BlobInfo storage.BlobInfo
}
func (s *Service) GetBlobInfo(ctx context.Context, args *GetBlobInfoArgs, reply *GetBlobInfoReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
blobInfo, err := bucket.Get(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetBlobInfoReply{
BlobInfo: &gob.BlobInfo{
Bucket_: blobInfo.Bucket(),
ContentType_: blobInfo.ContentType(),
BlobID_: blobInfo.ID(),
ModTime_: blobInfo.ModTime(),
Size_: blobInfo.Size(),
},
}
return nil
}

View File

@ -0,0 +1,33 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type GetBucketSizeArgs struct {
BucketID BucketID
}
type GetBucketSizeReply struct {
Size int64
}
func (s *Service) GetBucketSize(ctx context.Context, args *GetBucketSizeArgs, reply *GetBucketSizeReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
size, err := bucket.Size(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = GetBucketSizeReply{
Size: size,
}
return nil
}

View File

@ -0,0 +1,34 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type ListBlobInfoArgs struct {
BucketID BucketID
}
type ListBlobInfoReply struct {
BlobInfos []storage.BlobInfo
}
func (s *Service) ListBlobInfo(ctx context.Context, args *ListBlobInfoArgs, reply *ListBlobInfoReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
blobInfos, err := bucket.List(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = ListBlobInfoReply{
BlobInfos: blobInfos,
}
return nil
}

View File

@ -0,0 +1,27 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type ListBucketsArgs struct {
}
type ListBucketsReply struct {
Buckets []string
}
func (s *Service) ListBuckets(ctx context.Context, args *ListBucketsArgs, reply *ListBucketsReply) error {
buckets, err := s.store.ListBuckets(ctx)
if err != nil {
return errors.WithStack(err)
}
*reply = ListBucketsReply{
Buckets: buckets,
}
return nil
}

View File

@ -0,0 +1,57 @@
package blob
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type NewBlobReaderArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type NewBlobReaderReply struct {
ReaderID ReaderID
}
func (s *Service) NewBlobReader(ctx context.Context, args *NewBlobReaderArgs, reply *NewBlobReaderReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
readerID, err := NewReaderID()
if err != nil {
return errors.WithStack(err)
}
reader, err := bucket.NewReader(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
s.readers.Store(readerID, reader)
*reply = NewBlobReaderReply{
ReaderID: readerID,
}
return nil
}
func (s *Service) getOpenedReader(id ReaderID) (io.ReadSeekCloser, error) {
raw, exists := s.readers.Load(id)
if !exists {
return nil, errors.Errorf("could not find writer '%s'", id)
}
reader, ok := raw.(io.ReadSeekCloser)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for writer", raw)
}
return reader, nil
}

View File

@ -0,0 +1,57 @@
package blob
import (
"context"
"io"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type NewBlobWriterArgs struct {
BlobID storage.BlobID
BucketID BucketID
}
type NewBlobWriterReply struct {
WriterID WriterID
}
func (s *Service) NewBlobWriter(ctx context.Context, args *NewBlobWriterArgs, reply *NewBlobWriterReply) error {
bucket, err := s.getOpenedBucket(args.BucketID)
if err != nil {
return errors.WithStack(err)
}
writerID, err := NewWriterID()
if err != nil {
return errors.WithStack(err)
}
writer, err := bucket.NewWriter(ctx, args.BlobID)
if err != nil {
return errors.WithStack(err)
}
s.writers.Store(writerID, writer)
*reply = NewBlobWriterReply{
WriterID: writerID,
}
return nil
}
func (s *Service) getOpenedWriter(id WriterID) (io.WriteCloser, error) {
raw, exists := s.writers.Load(id)
if !exists {
return nil, errors.Errorf("could not find writer '%s'", id)
}
writer, ok := raw.(io.WriteCloser)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for writer", raw)
}
return writer, nil
}

View File

@ -0,0 +1,50 @@
package blob
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type OpenBucketArgs struct {
BucketName string
}
type OpenBucketReply struct {
BucketID BucketID
}
func (s *Service) OpenBucket(ctx context.Context, args *OpenBucketArgs, reply *OpenBucketReply) error {
bucket, err := s.store.OpenBucket(ctx, args.BucketName)
if err != nil {
return errors.WithStack(err)
}
bucketID, err := NewBucketID()
if err != nil {
return errors.WithStack(err)
}
s.buckets.Store(bucketID, bucket)
*reply = OpenBucketReply{
BucketID: bucketID,
}
return nil
}
func (s *Service) getOpenedBucket(id BucketID) (storage.BlobBucket, error) {
raw, exists := s.buckets.Load(id)
if !exists {
return nil, errors.WithStack(storage.ErrBucketClosed)
}
bucket, ok := raw.(storage.BlobBucket)
if !ok {
return nil, errors.Errorf("unexpected type '%T' for blob bucket", raw)
}
return bucket, nil
}

View File

@ -0,0 +1,41 @@
package blob
import (
"context"
"io"
"github.com/pkg/errors"
)
type ReadBlobArgs struct {
ReaderID ReaderID
Length int
}
type ReadBlobReply struct {
Data []byte
Read int
EOF bool
}
func (s *Service) ReadBlob(ctx context.Context, args *ReadBlobArgs, reply *ReadBlobReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
buff := make([]byte, args.Length)
read, err := reader.Read(buff)
if err != nil && !errors.Is(err, io.EOF) {
return errors.WithStack(err)
}
*reply = ReadBlobReply{
Read: read,
Data: buff,
EOF: errors.Is(err, io.EOF),
}
return nil
}

View File

@ -0,0 +1,38 @@
package blob
import (
"context"
"io"
"github.com/pkg/errors"
)
type SeekBlobArgs struct {
ReaderID ReaderID
Offset int64
Whence int
}
type SeekBlobReply struct {
Read int64
EOF bool
}
func (s *Service) SeekBlob(ctx context.Context, args *SeekBlobArgs, reply *SeekBlobReply) error {
reader, err := s.getOpenedReader(args.ReaderID)
if err != nil {
return errors.WithStack(err)
}
read, err := reader.Seek(args.Offset, args.Whence)
if err != nil && !errors.Is(err, io.EOF) {
return errors.WithStack(err)
}
*reply = SeekBlobReply{
Read: read,
EOF: errors.Is(err, io.EOF),
}
return nil
}

View File

@ -0,0 +1,60 @@
package blob
import (
"fmt"
"sync"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/google/uuid"
"github.com/pkg/errors"
)
type BucketID string
type WriterID string
type ReaderID string
type Service struct {
store storage.BlobStore
buckets sync.Map
writers sync.Map
readers sync.Map
}
func NewService(store storage.BlobStore) *Service {
return &Service{
store: store,
}
}
func NewBucketID() (BucketID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := BucketID(fmt.Sprintf("bucket-%s", uuid.String()))
return id, nil
}
func NewWriterID() (WriterID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := WriterID(fmt.Sprintf("writer-%s", uuid.String()))
return id, nil
}
func NewReaderID() (ReaderID, error) {
uuid, err := uuid.NewUUID()
if err != nil {
return "", errors.WithStack(err)
}
id := ReaderID(fmt.Sprintf("reader-%s", uuid.String()))
return id, nil
}

View File

@ -0,0 +1,34 @@
package blob
import (
"context"
"github.com/pkg/errors"
)
type WriteBlobArgs struct {
WriterID WriterID
Data []byte
}
type WriteBlobReply struct {
Written int
}
func (s *Service) WriteBlob(ctx context.Context, args *WriteBlobArgs, reply *WriteBlobReply) error {
writer, err := s.getOpenedWriter(args.WriterID)
if err != nil {
return errors.WithStack(err)
}
written, err := writer.Write(args.Data)
if err != nil {
return errors.WithStack(err)
}
*reply = WriteBlobReply{
Written: written,
}
return nil
}

View File

@ -0,0 +1,26 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type DeleteDocumentArgs struct {
Collection string
DocumentID storage.DocumentID
}
type DeleteDocumentReply struct {
}
func (s *Service) DeleteDocument(ctx context.Context, args DeleteDocumentArgs, reply *DeleteDocumentReply) error {
if err := s.store.Delete(ctx, args.Collection, args.DocumentID); err != nil {
return errors.WithStack(err)
}
*reply = DeleteDocumentReply{}
return nil
}

View File

@ -0,0 +1,30 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type GetDocumentArgs struct {
Collection string
DocumentID storage.DocumentID
}
type GetDocumentReply struct {
Document storage.Document
}
func (s *Service) GetDocument(ctx context.Context, args GetDocumentArgs, reply *GetDocumentReply) error {
document, err := s.store.Get(ctx, args.Collection, args.DocumentID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetDocumentReply{
Document: document,
}
return nil
}

View File

@ -0,0 +1,53 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/filter"
"github.com/pkg/errors"
)
type QueryDocumentsArgs struct {
Collection string
Filter map[string]any
Options *storage.QueryOptions
}
type QueryDocumentsReply struct {
Documents []storage.Document
}
func (s *Service) QueryDocuments(ctx context.Context, args QueryDocumentsArgs, reply *QueryDocumentsReply) error {
var (
argsFilter *filter.Filter
err error
)
if args.Filter != nil {
argsFilter, err = filter.NewFrom(args.Filter)
if err != nil {
return errors.WithStack(err)
}
}
documents, err := s.store.Query(ctx, args.Collection, argsFilter, withQueryOptions(args.Options))
if err != nil {
return errors.WithStack(err)
}
*reply = QueryDocumentsReply{
Documents: documents,
}
return nil
}
func withQueryOptions(opts *storage.QueryOptions) storage.QueryOptionFunc {
return func(o *storage.QueryOptions) {
o.Limit = opts.Limit
o.Offset = opts.Offset
o.OrderBy = opts.OrderBy
o.OrderDirection = opts.OrderDirection
}
}

View File

@ -0,0 +1,11 @@
package document
import "forge.cadoles.com/arcad/edge/pkg/storage"
type Service struct {
store storage.DocumentStore
}
func NewService(store storage.DocumentStore) *Service {
return &Service{store}
}

View File

@ -0,0 +1,30 @@
package document
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
)
type UpsertDocumentArgs struct {
Collection string
Document storage.Document
}
type UpsertDocumentReply struct {
Document storage.Document
}
func (s *Service) UpsertDocument(ctx context.Context, args UpsertDocumentArgs, reply *UpsertDocumentReply) error {
document, err := s.store.Upsert(ctx, args.Collection, args.Document)
if err != nil {
return errors.WithStack(err)
}
*reply = UpsertDocumentReply{
Document: document,
}
return nil
}

View File

@ -0,0 +1,5 @@
package server
import (
_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/gob"
)

View File

@ -0,0 +1,29 @@
package server
import (
"github.com/keegancsmith/rpc"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/blob"
"forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/document"
shareService "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc/server/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func NewBlobStoreServer(store storage.BlobStore) *rpc.Server {
server := rpc.NewServer()
server.Register(blob.NewService(store))
return server
}
func NewDocumentStoreServer(store storage.DocumentStore) *rpc.Server {
server := rpc.NewServer()
server.Register(document.NewService(store))
return server
}
func NewShareStoreServer(store share.Store) *rpc.Server {
server := rpc.NewServer()
server.Register(shareService.NewService(store))
return server
}

View File

@ -0,0 +1,28 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type DeleteAttributesArgs struct {
Origin app.ID
ResourceID share.ResourceID
Names []string
}
type DeleteAttributesReply struct {
}
func (s *Service) DeleteAttributes(ctx context.Context, args DeleteAttributesArgs, reply *DeleteAttributesReply) error {
if err := s.store.DeleteAttributes(ctx, args.Origin, args.ResourceID, args.Names...); err != nil {
return errors.WithStack(err)
}
*reply = DeleteAttributesReply{}
return nil
}

View File

@ -0,0 +1,27 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type DeleteResourceArgs struct {
Origin app.ID
ResourceID share.ResourceID
}
type DeleteResourceReply struct {
}
func (s *Service) DeleteResource(ctx context.Context, args DeleteResourceArgs, reply *DeleteResourceReply) error {
if err := s.store.DeleteResource(ctx, args.Origin, args.ResourceID); err != nil {
return errors.WithStack(err)
}
*reply = DeleteResourceReply{}
return nil
}

View File

@ -0,0 +1,41 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type FindResourcesArgs struct {
Options *share.FindResourcesOptions
}
type FindResourcesReply struct {
Resources []*SerializableResource
}
func (s *Service) FindResources(ctx context.Context, args FindResourcesArgs, reply *FindResourcesReply) error {
resources, err := s.store.FindResources(ctx, withFindResourcesOptions(args.Options))
if err != nil {
return errors.WithStack(err)
}
serializableResources := make([]*SerializableResource, len(resources))
for resIdx, r := range resources {
serializableResources[resIdx] = FromResource(r)
}
*reply = FindResourcesReply{
Resources: serializableResources,
}
return nil
}
func withFindResourcesOptions(opts *share.FindResourcesOptions) share.FindResourcesOptionFunc {
return func(o *share.FindResourcesOptions) {
o.Name = opts.Name
o.ValueType = opts.ValueType
}
}

View File

@ -0,0 +1,31 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type GetResourceArgs struct {
Origin app.ID
ResourceID share.ResourceID
}
type GetResourceReply struct {
Resource *SerializableResource
}
func (s *Service) GetResource(ctx context.Context, args GetResourceArgs, reply *GetResourceReply) error {
resource, err := s.store.GetResource(ctx, args.Origin, args.ResourceID)
if err != nil {
return errors.WithStack(err)
}
*reply = GetResourceReply{
Resource: FromResource(resource),
}
return nil
}

View File

@ -0,0 +1,94 @@
package share
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
func FromResource(res share.Resource) *SerializableResource {
serializableAttributes := make([]*SerializableAttribute, len(res.Attributes()))
for attrIdx, attr := range res.Attributes() {
serializableAttributes[attrIdx] = FromAttribute(attr)
}
return &SerializableResource{
ID_: res.ID(),
Origin_: res.Origin(),
Attributes_: serializableAttributes,
}
}
func FromAttribute(attr share.Attribute) *SerializableAttribute {
return &SerializableAttribute{
Name_: attr.Name(),
Value_: attr.Value(),
Type_: attr.Type(),
UpdatedAt_: attr.UpdatedAt(),
CreatedAt_: attr.CreatedAt(),
}
}
type SerializableResource struct {
ID_ share.ResourceID
Origin_ app.ID
Attributes_ []*SerializableAttribute
}
// Attributes implements share.Resource.
func (r *SerializableResource) Attributes() []share.Attribute {
attributes := make([]share.Attribute, len(r.Attributes_))
for idx, attr := range r.Attributes_ {
attributes[idx] = attr
}
return attributes
}
// ID implements share.Resource.
func (r *SerializableResource) ID() share.ResourceID {
return r.ID_
}
// Origin implements share.Resource.
func (r *SerializableResource) Origin() app.ID {
return r.Origin_
}
var _ share.Resource = &SerializableResource{}
type SerializableAttribute struct {
Name_ string
Value_ any
Type_ share.ValueType
UpdatedAt_ time.Time
CreatedAt_ time.Time
}
// CreatedAt implements share.Attribute.
func (a *SerializableAttribute) CreatedAt() time.Time {
return a.CreatedAt_
}
// Name implements share.Attribute.
func (a *SerializableAttribute) Name() string {
return a.Name_
}
// Type implements share.Attribute.
func (a *SerializableAttribute) Type() share.ValueType {
return a.Type_
}
// UpdatedAt implements share.Attribute.
func (a *SerializableAttribute) UpdatedAt() time.Time {
return a.UpdatedAt_
}
// Value implements share.Attribute.
func (a *SerializableAttribute) Value() any {
return a.Value_
}
var _ share.Attribute = &SerializableAttribute{}

View File

@ -0,0 +1,13 @@
package share
import (
"forge.cadoles.com/arcad/edge/pkg/storage/share"
)
type Service struct {
store share.Store
}
func NewService(store share.Store) *Service {
return &Service{store}
}

View File

@ -0,0 +1,37 @@
package share
import (
"context"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
type UpdateAttributesArgs struct {
Origin app.ID
ResourceID share.ResourceID
Attributes []*SerializableAttribute
}
type UpdateAttributesReply struct {
Resource *SerializableResource
}
func (s *Service) UpdateAttributes(ctx context.Context, args UpdateAttributesArgs, reply *UpdateAttributesReply) error {
attributes := make([]share.Attribute, len(args.Attributes))
for idx, attr := range args.Attributes {
attributes[idx] = attr
}
resource, err := s.store.UpdateAttributes(ctx, args.Origin, args.ResourceID, attributes...)
if err != nil {
return errors.WithStack(err)
}
*reply = UpdateAttributesReply{
Resource: FromResource(resource),
}
return nil
}

View File

@ -0,0 +1,35 @@
package driver
import (
"net/url"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
var shareStoreFactories = make(map[string]ShareStoreFactory, 0)
type ShareStoreFactory func(url *url.URL) (share.Store, error)
func RegisterShareStoreFactory(scheme string, factory ShareStoreFactory) {
shareStoreFactories[scheme] = factory
}
func NewShareStore(dsn string) (share.Store, error) {
url, err := url.Parse(dsn)
if err != nil {
return nil, errors.WithStack(err)
}
factory, exists := shareStoreFactories[url.Scheme]
if !exists {
return nil, errors.WithStack(ErrSchemeNotRegistered)
}
store, err := factory(url)
if err != nil {
return nil, errors.WithStack(err)
}
return store, nil
}

View File

@ -0,0 +1,459 @@
package sqlite
import (
"bytes"
"context"
"database/sql"
"io"
"sync"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/gabriel-vasile/mimetype"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type BlobBucket struct {
name string
getDB GetDBFunc
closed bool
}
// Size implements storage.BlobBucket
func (b *BlobBucket) Size(ctx context.Context) (int64, error) {
var size int64
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT SUM(size) FROM blobs WHERE bucket = $1`
row := tx.QueryRowContext(ctx, query, b.name)
var nullSize sql.NullInt64
if err := row.Scan(&nullSize); err != nil {
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
size = nullSize.Int64
return nil
})
if err != nil {
return 0, errors.WithStack(err)
}
return size, nil
}
// Name implements storage.BlobBucket
func (b *BlobBucket) Name() string {
return b.name
}
// Close implements storage.BlobBucket
func (b *BlobBucket) Close() error {
logger.Debug(
context.Background(), "closing bucket",
logger.F("alreadyClosed", b.closed),
logger.F("name", b.name),
)
b.closed = true
return nil
}
// Delete implements storage.BlobBucket
func (b *BlobBucket) Delete(ctx context.Context, id storage.BlobID) error {
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `DELETE FROM blobs WHERE bucket = $1 AND id = $2`
args := []any{b.name, id}
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
if _, err := tx.ExecContext(ctx, query, args...); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.BlobBucket
func (b *BlobBucket) Get(ctx context.Context, id storage.BlobID) (storage.BlobInfo, error) {
var blobInfo *BlobInfo
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT content_type, mod_time, size FROM blobs WHERE bucket = $1 AND id = $2`
args := []any{b.name, id}
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
row := tx.QueryRowContext(ctx, query, args...)
var (
contentType string
modTime time.Time
size int64
)
if err := row.Scan(&contentType, &modTime, &size); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return errors.WithStack(storage.ErrBlobNotFound)
}
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
blobInfo = &BlobInfo{
id: id,
bucket: b.name,
contentType: contentType,
modTime: modTime,
size: size,
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return blobInfo, nil
}
// List implements storage.BlobBucket
func (b *BlobBucket) List(ctx context.Context) ([]storage.BlobInfo, error) {
var blobs []storage.BlobInfo
err := b.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT id, content_type, mod_time, size FROM blobs WHERE bucket = $1`
args := []any{b.name}
logger.Debug(ctx, "executing query", logger.F("query", query), logger.F("args", args))
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
blobs = make([]storage.BlobInfo, 0)
for rows.Next() {
var (
blobID string
contentType string
modTime time.Time
size int64
)
if err := rows.Scan(&blobID, &contentType, &modTime, &size); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return errors.WithStack(storage.ErrBlobNotFound)
}
return errors.WithStack(err)
}
blobInfo := &BlobInfo{
id: storage.BlobID(blobID),
bucket: b.name,
contentType: contentType,
modTime: modTime,
size: size,
}
blobs = append(blobs, blobInfo)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return blobs, nil
}
// NewReader implements storage.BlobBucket
func (b *BlobBucket) NewReader(ctx context.Context, id storage.BlobID) (io.ReadSeekCloser, error) {
if b.closed {
return nil, errors.WithStack(storage.ErrBucketClosed)
}
return &blobReaderCloser{
id: id,
bucket: b.name,
getDB: b.getDB,
}, nil
}
// NewWriter implements storage.BlobBucket
func (b *BlobBucket) NewWriter(ctx context.Context, id storage.BlobID) (io.WriteCloser, error) {
if b.closed {
return nil, errors.WithStack(storage.ErrBucketClosed)
}
return &blobWriterCloser{
id: id,
bucket: b.name,
getDB: b.getDB,
buf: bytes.Buffer{},
}, nil
}
func (b *BlobBucket) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
if b.closed {
return errors.WithStack(storage.ErrBucketClosed)
}
db, err := b.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
type blobWriterCloser struct {
id storage.BlobID
bucket string
getDB GetDBFunc
buf bytes.Buffer
closed bool
}
// Write implements io.WriteCloser
func (wbc *blobWriterCloser) Write(p []byte) (int, error) {
logger.Debug(
context.Background(), "writing data to blob",
logger.F("size", len(p)),
logger.F("blobID", wbc.id),
logger.F("bucket", wbc.bucket),
)
n, err := wbc.buf.Write(p)
if err != nil {
return n, errors.WithStack(err)
}
return n, nil
}
// Close implements io.WriteCloser
func (wbc *blobWriterCloser) Close() error {
ctx := context.Background()
logger.Debug(
ctx, "closing writer",
logger.F("alreadyClosed", wbc.closed),
logger.F("bucket", wbc.bucket),
logger.F("blobID", wbc.id),
)
if wbc.closed {
return nil
}
err := wbc.withTx(ctx, func(tx *sql.Tx) error {
query := `
INSERT INTO blobs (bucket, id, data, content_type, mod_time, size)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (id, bucket) DO UPDATE SET
data = $3, content_type = $4, mod_time = $5, size = $6
`
data := wbc.buf.Bytes()
mime := mimetype.Detect(data)
modTime := time.Now().UTC()
args := []any{
wbc.bucket,
wbc.id,
data,
mime.String(),
modTime,
len(data),
}
logger.Debug(ctx, "executing query", logger.F("query", query))
_, err := tx.Exec(
query,
args...,
)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
wbc.closed = true
return nil
}
func (wbc *blobWriterCloser) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
if wbc.closed {
return errors.WithStack(io.ErrClosedPipe)
}
db, err := wbc.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
type blobReaderCloser struct {
id storage.BlobID
bucket string
getDB GetDBFunc
reader bytes.Reader
once sync.Once
closed bool
}
// Read implements io.ReadSeekCloser
func (brc *blobReaderCloser) Read(p []byte) (int, error) {
var err error
brc.once.Do(func() {
err = brc.loadBlob()
})
if err != nil {
return 0, errors.WithStack(err)
}
n, err := brc.reader.Read(p)
if err != nil {
if errors.Is(err, io.EOF) {
return n, io.EOF
}
return n, errors.WithStack(err)
}
return n, nil
}
// Seek implements io.ReadSeekCloser
func (brc *blobReaderCloser) Seek(offset int64, whence int) (int64, error) {
var err error
brc.once.Do(func() {
err = brc.loadBlob()
})
if err != nil {
return 0, errors.WithStack(err)
}
n, err := brc.reader.Seek(offset, whence)
if err != nil {
return n, errors.WithStack(err)
}
return n, nil
}
func (brc *blobReaderCloser) loadBlob() error {
ctx := context.Background()
logger.Debug(ctx, "loading blob", logger.F("alreadyClosed", brc.closed))
err := brc.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT data FROM blobs WHERE bucket = $1 AND id = $2`
row := tx.QueryRow(query, brc.bucket, brc.id)
var data []byte
if err := row.Scan(&data); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return errors.WithStack(storage.ErrBlobNotFound)
}
return errors.WithStack(err)
}
brc.reader = *bytes.NewReader(data)
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
// Close implements io.ReadSeekCloser
func (brc *blobReaderCloser) Close() error {
logger.Debug(
context.Background(), "closing reader",
logger.F("alreadyClosed", brc.closed),
logger.F("bucket", brc.bucket),
logger.F("blobID", brc.id),
)
brc.closed = true
return nil
}
func (brc *blobReaderCloser) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
db, err := brc.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
var (
_ storage.BlobBucket = &BlobBucket{}
_ storage.BlobInfo = &BlobInfo{}
_ io.WriteCloser = &blobWriterCloser{}
_ io.ReadSeekCloser = &blobReaderCloser{}
)

View File

@ -0,0 +1,40 @@
package sqlite
import (
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
)
type BlobInfo struct {
id storage.BlobID
bucket string
contentType string
modTime time.Time
size int64
}
// Bucket implements storage.BlobInfo
func (i *BlobInfo) Bucket() string {
return i.bucket
}
// ID implements storage.BlobInfo
func (i *BlobInfo) ID() storage.BlobID {
return i.id
}
// ContentType implements storage.BlobInfo
func (i *BlobInfo) ContentType() string {
return i.contentType
}
// ModTime implements storage.BlobInfo
func (i *BlobInfo) ModTime() time.Time {
return i.modTime
}
// Size implements storage.BlobInfo
func (i *BlobInfo) Size() int64 {
return i.size
}

View File

@ -0,0 +1,136 @@
package sqlite
import (
"context"
"database/sql"
"forge.cadoles.com/arcad/edge/pkg/storage"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type BlobStore struct {
getDB GetDBFunc
}
// DeleteBucket implements storage.BlobStore
func (s *BlobStore) DeleteBucket(ctx context.Context, name string) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `DELETE FROM blobs WHERE bucket = $1`
_, err := tx.ExecContext(ctx, query, name)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
// ListBuckets implements storage.BlobStore
func (s *BlobStore) ListBuckets(ctx context.Context) ([]string, error) {
buckets := make([]string, 0)
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `SELECT DISTINCT bucket FROM blobs`
rows, err := tx.QueryContext(ctx, query)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return errors.WithStack(err)
}
buckets = append(buckets, name)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return buckets, nil
}
// OpenBucket implements storage.BlobStore
func (s *BlobStore) OpenBucket(ctx context.Context, name string) (storage.BlobBucket, error) {
return &BlobBucket{
name: name,
getDB: s.getDB,
}, nil
}
func ensureBlobTables(ctx context.Context, db *sql.DB) error {
logger.Debug(ctx, "creating blobs table")
err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := `
CREATE TABLE IF NOT EXISTS blobs (
id TEXT,
bucket TEXT,
data BLOB,
content_type TEXT NOT NULL,
mod_time TIMESTAMP NOT NULL,
size INTEGER,
PRIMARY KEY (id, bucket)
);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func (s *BlobStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
var db *sql.DB
db, err := s.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
func NewBlobStore(dsn string) *BlobStore {
getDB := NewGetDBFunc(dsn, ensureBlobTables)
return &BlobStore{getDB}
}
func NewBlobStoreWithDB(db *sql.DB) *BlobStore {
getDB := NewGetDBFuncFromDB(db, ensureBlobTables)
return &BlobStore{getDB}
}
var _ storage.BlobStore = &BlobStore{}

View File

@ -0,0 +1,46 @@
package sqlite
import (
"context"
"fmt"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestBlobStore(t *testing.T) {
t.Parallel()
if testing.Verbose() {
logger.SetLevel(logger.LevelDebug)
}
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%+v", errors.WithStack(err))
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewBlobStore(dsn)
testsuite.TestBlobStore(context.Background(), t, store)
}
func BenchmarkBlobStore(t *testing.B) {
logger.SetLevel(logger.LevelError)
file := "./testdata/blobstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%+v", errors.WithStack(err))
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewBlobStore(dsn)
testsuite.BenchmarkBlobStore(t, store)
}

View File

@ -0,0 +1,362 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"math"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/filter"
filterSQL "forge.cadoles.com/arcad/edge/pkg/storage/filter/sql"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
_ "modernc.org/sqlite"
)
type DocumentStore struct {
getDB GetDBFunc
}
// Delete implements storage.DocumentStore
func (s *DocumentStore) Delete(ctx context.Context, collection string, id storage.DocumentID) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
DELETE FROM documents
WHERE collection = $1 AND id = $2
`
_, err := tx.ExecContext(ctx, query, collection, string(id))
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
// Get implements storage.DocumentStore
func (s *DocumentStore) Get(ctx context.Context, collection string, id storage.DocumentID) (storage.Document, error) {
var document storage.Document
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
SELECT id, data, created_at, updated_at
FROM documents
WHERE collection = $1 AND id = $2
`
row := tx.QueryRowContext(ctx, query, collection, string(id))
var (
createdAt time.Time
updatedAt time.Time
data JSONMap
)
err := row.Scan(&id, &data, &createdAt, &updatedAt)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return errors.WithStack(storage.ErrDocumentNotFound)
}
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
document = storage.Document(data)
document[storage.DocumentAttrID] = id
document[storage.DocumentAttrCreatedAt] = createdAt
document[storage.DocumentAttrUpdatedAt] = updatedAt
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return document, nil
}
// Query implements storage.DocumentStore
func (s *DocumentStore) Query(ctx context.Context, collection string, filter *filter.Filter, funcs ...storage.QueryOptionFunc) ([]storage.Document, error) {
opts := &storage.QueryOptions{}
for _, fn := range funcs {
fn(opts)
}
var documents []storage.Document
err := s.withTx(ctx, func(tx *sql.Tx) error {
criteria := "1 = 1"
args := make([]any, 0)
var err error
if filter != nil {
criteria, args, err = filterSQL.ToSQL(
filter.Root(),
filterSQL.WithPreparedParameter("$", 2),
filterSQL.WithTransform(transformOperator),
filterSQL.WithKeyTransform(func(key string) string {
return fmt.Sprintf("json_extract(data, '$.%s')", key)
}),
)
if err != nil {
return errors.WithStack(err)
}
}
query := `
SELECT id, data, created_at, updated_at
FROM documents
WHERE collection = $1 AND (` + criteria + `)
`
args = append([]interface{}{collection}, args...)
if opts.OrderBy != nil {
direction := storage.OrderDirectionAsc
if opts.OrderDirection != nil {
direction = *opts.OrderDirection
}
query, args = withOrderByClause(query, args, *opts.OrderBy, direction)
}
if opts.Offset != nil || opts.Limit != nil {
offset := 0
if opts.Offset != nil {
offset = *opts.Offset
}
limit := math.MaxInt
if opts.Limit != nil {
limit = *opts.Limit
}
query, args = withLimitOffsetClause(query, args, limit, offset)
}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
documents = make([]storage.Document, 0)
for rows.Next() {
var (
id storage.DocumentID
createdAt time.Time
updatedAt time.Time
data JSONMap
)
if err := rows.Scan(&id, &data, &createdAt, &updatedAt); err != nil {
return errors.WithStack(err)
}
document := storage.Document(data)
document[storage.DocumentAttrID] = id
document[storage.DocumentAttrCreatedAt] = createdAt
document[storage.DocumentAttrUpdatedAt] = updatedAt
documents = append(documents, document)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return documents, nil
}
// Upsert implements storage.DocumentStore
func (s *DocumentStore) Upsert(ctx context.Context, collection string, document storage.Document) (storage.Document, error) {
var upsertedDocument storage.Document
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
INSERT INTO documents (id, collection, data, created_at, updated_at)
VALUES($1, $2, $3, $4, $4)
ON CONFLICT (id, collection) DO UPDATE SET
data = $3, updated_at = $4
RETURNING "id", "data", "created_at", "updated_at"
`
now := time.Now().UTC()
id, exists := document.ID()
if !exists || id == "" {
id = storage.NewDocumentID()
}
args := []any{id, collection, JSONMap(document), now, now}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
row := tx.QueryRowContext(ctx, query, args...)
var (
createdAt time.Time
updatedAt time.Time
data JSONMap
)
err := row.Scan(&id, &data, &createdAt, &updatedAt)
if err != nil {
return errors.WithStack(err)
}
if err := row.Err(); err != nil {
return errors.WithStack(err)
}
upsertedDocument = storage.Document(data)
upsertedDocument[storage.DocumentAttrID] = id
upsertedDocument[storage.DocumentAttrCreatedAt] = createdAt
upsertedDocument[storage.DocumentAttrUpdatedAt] = updatedAt
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return upsertedDocument, nil
}
func (s *DocumentStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
var db *sql.DB
db, err := s.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
func ensureDocumentTables(ctx context.Context, db *sql.DB) error {
err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := `
CREATE TABLE IF NOT EXISTS documents (
id TEXT PRIMARY KEY,
collection TEXT NOT NULL,
data TEXT,
created_at TIMESTAMP NOT NULL,
updated_at TIMESTAMP NOT NULL,
UNIQUE(id, collection) ON CONFLICT REPLACE
);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
query = `
CREATE INDEX IF NOT EXISTS collection_idx ON documents (collection);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func withOrderByClause(query string, args []any, orderBy string, orderDirection storage.OrderDirection) (string, []any) {
direction := "ASC"
if orderDirection == storage.OrderDirectionDesc {
direction = "DESC"
}
var column string
switch orderBy {
case storage.DocumentAttrID:
column = "id"
case storage.DocumentAttrCreatedAt:
column = "created_at"
case storage.DocumentAttrUpdatedAt:
column = "updated_at"
default:
column = fmt.Sprintf("json_extract(data, '$.' || $%d)", len(args)+1)
args = append(args, orderBy)
}
query += fmt.Sprintf(` ORDER BY %s %s`, column, direction)
return query, args
}
func withLimitOffsetClause(query string, args []any, limit int, offset int) (string, []any) {
query += fmt.Sprintf(` LIMIT $%d OFFSET $%d`, len(args)+1, len(args)+2)
args = append(args, limit, offset)
return query, args
}
func NewDocumentStore(path string) *DocumentStore {
getDB := NewGetDBFunc(path, ensureDocumentTables)
return &DocumentStore{
getDB: getDB,
}
}
func NewDocumentStoreWithDB(db *sql.DB) *DocumentStore {
getDB := NewGetDBFuncFromDB(db, ensureDocumentTables)
return &DocumentStore{
getDB: getDB,
}
}
var _ storage.DocumentStore = &DocumentStore{}

View File

@ -0,0 +1,29 @@
package sqlite
import (
"context"
"fmt"
"os"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestDocumentStore(t *testing.T) {
t.Parallel()
logger.SetLevel(logger.LevelDebug)
file := "./testdata/documentstore_test.sqlite"
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("%+v", errors.WithStack(err))
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewDocumentStore(dsn)
testsuite.TestDocumentStore(context.Background(), t, store)
}

View File

@ -0,0 +1,75 @@
package sqlite
import (
"net/url"
"os"
"path/filepath"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/driver"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
)
func init() {
driver.RegisterDocumentStoreFactory("sqlite", documentStoreFactory)
driver.RegisterBlobStoreFactory("sqlite", blobStoreFactory)
driver.RegisterShareStoreFactory("sqlite", shareStoreFactory)
}
func documentStoreFactory(url *url.URL) (storage.DocumentStore, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewDocumentStoreWithDB(db), nil
}
func blobStoreFactory(url *url.URL) (storage.BlobStore, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewBlobStoreWithDB(db), nil
}
func shareStoreFactory(url *url.URL) (share.Store, error) {
dir := filepath.Dir(url.Host + url.Path)
if dir != ":memory:" {
if err := os.MkdirAll(dir, os.FileMode(0750)); err != nil {
return nil, errors.WithStack(err)
}
}
path := url.Host + url.Path + "?" + url.RawQuery
db, err := Open(path)
if err != nil {
return nil, errors.WithStack(err)
}
return NewShareStoreWithDB(db), nil
}

View File

@ -0,0 +1,50 @@
package sqlite
import (
"fmt"
"forge.cadoles.com/arcad/edge/pkg/storage"
"forge.cadoles.com/arcad/edge/pkg/storage/filter/sql"
)
func transformOperator(operator string, invert bool, key string, value any, option *sql.Option) (string, any, error) {
isDataAttr := true
switch key {
case storage.DocumentAttrCreatedAt:
key = "created_at"
isDataAttr = false
case storage.DocumentAttrUpdatedAt:
key = "updated_at"
isDataAttr = false
case storage.DocumentAttrID:
key = "id"
isDataAttr = false
}
if !isDataAttr {
option = &sql.Option{
PreparedParameter: option.PreparedParameter,
KeyTransform: func(key string) string {
return key
},
ValueTransform: option.ValueTransform,
Transform: option.Transform,
}
}
switch operator {
case sql.OpIn:
return transformInOperator(key, value, option)
default:
return sql.DefaultTransform(operator, invert, key, value, option)
}
}
func transformInOperator(key string, value any, option *sql.Option) (string, any, error) {
return fmt.Sprintf(
"EXISTS (SELECT 1 FROM json_each(json_extract(data, \"$.%v\")) WHERE value = %v)",
key,
option.PreparedParameter(),
), option.ValueTransform(value), nil
}

View File

@ -0,0 +1,42 @@
package sqlite
import (
"database/sql/driver"
"encoding/json"
"github.com/pkg/errors"
)
type JSONMap map[string]any
func (j *JSONMap) Scan(value interface{}) error {
if value == nil {
return nil
}
var data []byte
switch typ := value.(type) {
case []byte:
data = typ
case string:
data = []byte(typ)
default:
return errors.Errorf("unexpected type '%T'", value)
}
if err := json.Unmarshal(data, &j); err != nil {
return errors.WithStack(err)
}
return nil
}
func (j JSONMap) Value() (driver.Value, error) {
data, err := json.Marshal(j)
if err != nil {
return nil, errors.WithStack(err)
}
return data, nil
}

View File

@ -0,0 +1,428 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"time"
"forge.cadoles.com/arcad/edge/pkg/app"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
type ShareStore struct {
getDB GetDBFunc
}
// DeleteAttributes implements share.Repository
func (s *ShareStore) DeleteAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, names ...string) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
DELETE FROM resources
WHERE origin = $1 AND resource_id = $2
`
args := []any{origin, resourceID}
criteria := ""
for idx, name := range names {
if idx == 0 {
criteria += " AND ("
}
if idx != 0 {
criteria += " OR "
}
criteria += fmt.Sprintf(" name = $%d", len(args)+1)
args = append(args, name)
if idx == len(names)-1 {
criteria += " )"
}
}
query += criteria
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
res, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
affected, err := res.RowsAffected()
if err != nil {
return errors.WithStack(err)
}
if affected == 0 {
return errors.WithStack(share.ErrNotFound)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return err
}
// DeleteResource implements share.Repository
func (s *ShareStore) DeleteResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) error {
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
DELETE FROM resources
WHERE origin = $1 AND resource_id = $2
`
args := []any{origin, resourceID}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
res, err := tx.ExecContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
affected, err := res.RowsAffected()
if err != nil {
return errors.WithStack(err)
}
if affected == 0 {
return errors.WithStack(share.ErrNotFound)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return err
}
// FindResources implements share.Repository
func (s *ShareStore) FindResources(ctx context.Context, funcs ...share.FindResourcesOptionFunc) ([]share.Resource, error) {
opts := share.NewFindResourcesOptions(funcs...)
var resources []share.Resource
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
SELECT
main.origin, main.resource_id,
main.name, main.type, main.value,
main.created_at, main.updated_at
FROM resources AS main
JOIN resources AS sub ON
main.resource_id = sub.resource_id
AND main.origin = sub.origin
`
criteria := " WHERE 1 = 1"
preparedArgIndex := 1
args := make([]any, 0)
if opts.Name != nil {
criteria += fmt.Sprintf(" AND sub.name = $%d", preparedArgIndex)
args = append(args, *opts.Name)
preparedArgIndex++
}
if opts.ValueType != nil {
criteria += fmt.Sprintf(" AND sub.type = $%d", preparedArgIndex)
args = append(args, *opts.ValueType)
preparedArgIndex++
}
query += criteria
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
rows, err := tx.QueryContext(ctx, query, args...)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
indexedResources := make(map[string]*share.BaseResource)
for rows.Next() {
var (
origin string
resourceID string
name string
valueType string
value any
updatedAt time.Time
createdAt time.Time
)
if err := rows.Scan(&origin, &resourceID, &name, &valueType, &value, &createdAt, &updatedAt); err != nil {
return errors.WithStack(err)
}
resourceKey := origin + resourceID
resource, exists := indexedResources[resourceKey]
if !exists {
resource = share.NewBaseResource(app.ID(origin), share.ResourceID(resourceID))
indexedResources[resourceKey] = resource
}
attr := share.NewBaseAttribute(
name,
share.ValueType(valueType),
value,
)
attr.SetCreatedAt(createdAt)
attr.SetUpdatedAt(updatedAt)
resource.SetAttribute(attr)
}
if err := rows.Err(); err != nil {
return errors.WithStack(err)
}
resources = make([]share.Resource, 0, len(indexedResources))
for _, res := range indexedResources {
resources = append(resources, res)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resources, nil
}
// GetResource implements share.Repository
func (s *ShareStore) GetResource(ctx context.Context, origin app.ID, resourceID share.ResourceID) (share.Resource, error) {
var (
resource *share.BaseResource
err error
)
err = s.withTx(ctx, func(tx *sql.Tx) error {
resource, err = s.getResourceWithinTx(ctx, tx, origin, resourceID)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resource, nil
}
// UpdateAttributes implements share.Repository
func (s *ShareStore) UpdateAttributes(ctx context.Context, origin app.ID, resourceID share.ResourceID, attributes ...share.Attribute) (share.Resource, error) {
if len(attributes) == 0 {
return nil, errors.WithStack(share.ErrAttributeRequired)
}
var resource *share.BaseResource
err := s.withTx(ctx, func(tx *sql.Tx) error {
query := `
INSERT INTO resources (origin, resource_id, name, type, value, created_at, updated_at)
VALUES($1, $2, $3, $4, $5, $6, $6)
ON CONFLICT (origin, resource_id, name) DO UPDATE SET
type = $4, value = $5, updated_at = $6
`
stmt, err := tx.PrepareContext(ctx, query)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := stmt.Close(); err != nil {
logger.Error(ctx, "could not close statement", logger.E(errors.WithStack(err)))
}
}()
now := time.Now().UTC()
for _, attr := range attributes {
args := []any{
string(origin), string(resourceID),
attr.Name(), string(attr.Type()), attr.Value(),
now, now,
}
logger.Debug(
ctx, "executing query",
logger.F("query", query),
logger.F("args", args),
)
if _, err := stmt.ExecContext(ctx, args...); err != nil {
return errors.WithStack(err)
}
}
resource, err = s.getResourceWithinTx(ctx, tx, origin, resourceID)
if err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return nil, errors.WithStack(err)
}
return resource, nil
}
func (s *ShareStore) getResourceWithinTx(ctx context.Context, tx *sql.Tx, origin app.ID, resourceID share.ResourceID) (*share.BaseResource, error) {
query := `
SELECT name, type, value, created_at, updated_at
FROM resources
WHERE origin = $1 AND resource_id = $2
`
rows, err := tx.QueryContext(ctx, query, origin, resourceID)
if err != nil {
return nil, errors.WithStack(err)
}
defer func() {
if err := rows.Close(); err != nil {
logger.Error(ctx, "could not close rows", logger.E(errors.WithStack(err)))
}
}()
attributes := make([]share.Attribute, 0)
for rows.Next() {
var (
name string
valueType string
value any
updatedAt time.Time
createdAt time.Time
)
if err := rows.Scan(&name, &valueType, &value, &createdAt, &updatedAt); err != nil {
return nil, errors.WithStack(err)
}
attr := share.NewBaseAttribute(
name,
share.ValueType(valueType),
value,
)
attr.SetCreatedAt(createdAt)
attr.SetUpdatedAt(updatedAt)
attributes = append(attributes, attr)
}
if err := rows.Err(); err != nil {
return nil, errors.WithStack(err)
}
if len(attributes) == 0 {
return nil, errors.WithStack(share.ErrNotFound)
}
resource := share.NewBaseResource(origin, resourceID, attributes...)
return resource, nil
}
func (s *ShareStore) withTx(ctx context.Context, fn func(tx *sql.Tx) error) error {
var db *sql.DB
db, err := s.getDB(ctx)
if err != nil {
return errors.WithStack(err)
}
if err := WithTx(ctx, db, fn); err != nil {
return errors.WithStack(err)
}
return nil
}
func ensureShareTables(ctx context.Context, db *sql.DB) error {
err := WithTx(ctx, db, func(tx *sql.Tx) error {
query := `
CREATE TABLE IF NOT EXISTS resources (
resource_id TEXT NOT NULL,
origin TEXT NOT NULL,
name TEXT NOT NULL,
type TEXT NOT NULL,
value TEXT,
created_at TIMESTAMP NOT NULL,
updated_at TIMESTAMP NOT NULL,
UNIQUE(origin, resource_id, name) ON CONFLICT REPLACE
);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
query = `
CREATE INDEX IF NOT EXISTS resource_idx ON resources (origin, resource_id, name);
`
if _, err := tx.ExecContext(ctx, query); err != nil {
return errors.WithStack(err)
}
return nil
})
if err != nil {
return errors.WithStack(err)
}
return nil
}
func NewShareStore(path string) *ShareStore {
getDB := NewGetDBFunc(path, ensureShareTables)
return &ShareStore{
getDB: getDB,
}
}
func NewShareStoreWithDB(db *sql.DB) *ShareStore {
getDB := NewGetDBFuncFromDB(db, ensureShareTables)
return &ShareStore{
getDB: getDB,
}
}
var _ share.Store = &ShareStore{}

View File

@ -0,0 +1,33 @@
package sqlite
import (
"fmt"
"os"
"strings"
"testing"
"time"
"forge.cadoles.com/arcad/edge/pkg/storage/share"
"forge.cadoles.com/arcad/edge/pkg/storage/share/testsuite"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
)
func TestRepository(t *testing.T) {
logger.SetLevel(logger.LevelDebug)
testsuite.TestStore(t, newTestStore)
}
func newTestStore(testName string) (share.Store, error) {
filename := strings.ToLower(strings.ReplaceAll(testName, " ", "_"))
file := fmt.Sprintf("./testdata/%s.sqlite", filename)
if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, errors.WithStack(err)
}
dsn := fmt.Sprintf("%s?_pragma=foreign_keys(1)&_pragma=busy_timeout=%d", file, (60 * time.Second).Milliseconds())
store := NewShareStore(dsn)
return store, nil
}

View File

@ -0,0 +1,131 @@
package sqlite
import (
"context"
"database/sql"
"sync"
"github.com/pkg/errors"
"gitlab.com/wpetit/goweb/logger"
"modernc.org/sqlite"
_ "modernc.org/sqlite"
sqlite3 "modernc.org/sqlite/lib"
)
func Open(path string) (*sql.DB, error) {
db, err := sql.Open("sqlite", path)
if err != nil {
return nil, errors.Wrapf(err, "could not open database with path '%s'", path)
}
return db, nil
}
func WithTx(ctx context.Context, db *sql.DB, fn func(tx *sql.Tx) error) error {
var tx *sql.Tx
tx, err := db.BeginTx(ctx, nil)
if err != nil {
return errors.WithStack(err)
}
defer func() {
if err := tx.Rollback(); err != nil {
if errors.Is(err, sql.ErrTxDone) {
return
}
panic(errors.WithStack(err))
}
}()
for {
if err = fn(tx); err != nil {
var sqlErr *sqlite.Error
if errors.As(err, &sqlErr) {
if sqlErr.Code() == sqlite3.SQLITE_BUSY {
logger.Warn(ctx, "database busy, retrying transaction")
if err := ctx.Err(); err != nil {
logger.Error(ctx, "could not execute transaction", logger.E(errors.WithStack(err)))
return errors.WithStack(err)
}
continue
}
}
return errors.WithStack(err)
}
break
}
if err = tx.Commit(); err != nil {
return errors.WithStack(err)
}
return nil
}
type GetDBFunc func(ctx context.Context) (*sql.DB, error)
func NewGetDBFunc(dsn string, initFunc func(ctx context.Context, db *sql.DB) error) GetDBFunc {
var (
db *sql.DB
mutex sync.RWMutex
)
return func(ctx context.Context) (*sql.DB, error) {
mutex.RLock()
if db != nil {
defer mutex.RUnlock()
return db, nil
}
mutex.RUnlock()
mutex.Lock()
defer mutex.Unlock()
logger.Debug(ctx, "opening database", logger.F("dsn", dsn))
newDB, err := sql.Open("sqlite", dsn)
if err != nil {
return nil, errors.WithStack(err)
}
logger.Debug(ctx, "initializing database")
if err = initFunc(ctx, newDB); err != nil {
return nil, errors.WithStack(err)
}
db = newDB
return db, nil
}
}
func NewGetDBFuncFromDB(db *sql.DB, initFunc func(ctx context.Context, db *sql.DB) error) GetDBFunc {
var err error
initOnce := &sync.Once{}
return func(ctx context.Context) (*sql.DB, error) {
initOnce.Do(func() {
logger.Debug(ctx, "initializing database")
err = initFunc(ctx, db)
})
if err != nil {
return nil, errors.WithStack(err)
}
return db, nil
}
}

View File

@ -0,0 +1 @@
/*.sqlite*