edge/cmd/blobstore-test/main.go

package main

import (
	"context"
	"crypto/rand"
	"flag"
	"io"
	mrand "math/rand"
	"runtime"
	"time"

	"forge.cadoles.com/arcad/edge/pkg/storage"
	"forge.cadoles.com/arcad/edge/pkg/storage/driver"
	"github.com/pkg/errors"
	"gitlab.com/wpetit/goweb/logger"

	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/cache"
	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/rpc"
	_ "forge.cadoles.com/arcad/edge/pkg/storage/driver/sqlite"
)

var (
	dsn string
)

func init() {
	flag.StringVar(&dsn, "dsn", "cache://./test-cache.sqlite?driver=sqlite&_pragma=foreign_keys(1)&_pragma=journal_mode=wal&bigCacheShards=32&bigCacheHardMaxCacheSize=128&bigCacheMaxEntrySize=125&bigCacheMaxEntriesInWindow=200000", "blobstore dsn")
}

func main() {
	flag.Parse()

	ctx := context.Background()

	logger.SetLevel(logger.LevelDebug)

	blobStore, err := driver.NewBlobStore(dsn)
	if err != nil {
		logger.Fatal(ctx, "could not create blobstore", logger.CapturedE(errors.WithStack(err)))
	}

	bucket, err := blobStore.OpenBucket(ctx, "default")
	if err != nil {
		logger.Fatal(ctx, "could not open bucket", logger.CapturedE(errors.WithStack(err)))
	}

	defer func() {
		if err := bucket.Close(); err != nil {
			logger.Fatal(ctx, "could not close bucket", logger.CapturedE(errors.WithStack(err)))
		}
	}()

	go readRandomBlobs(ctx, bucket)

	for {
		writeRandomBlob(ctx, bucket)

		time.Sleep(1 * time.Second)

		size, err := bucket.Size(ctx)
		if err != nil {
			logger.Fatal(ctx, "could not retrieve bucket size", logger.CapturedE(errors.WithStack(err)))
		}

		logger.Debug(ctx, "bucket stats", logger.F("size", size))
	}
}

func readRandomBlobs(ctx context.Context, bucket storage.BlobBucket) {
	for {
		infos, err := bucket.List(ctx)
		if err != nil {
			logger.Fatal(ctx, "could not list blobs", logger.CapturedE(errors.WithStack(err)))
		}

		total := len(infos)
		if total == 0 {
			logger.Debug(ctx, "no blob yet")

			continue
		}

		blob := infos[mrand.Intn(total)]

		readBlob(ctx, bucket, blob.ID())

		time.Sleep(250 * time.Millisecond)
	}
}

func readBlob(ctx context.Context, bucket storage.BlobBucket, blobID storage.BlobID) {
	ctx = logger.With(ctx, logger.F("blobID", blobID))

	reader, err := bucket.NewReader(ctx, blobID)
	if err != nil {
		logger.Fatal(ctx, "could not create reader", logger.CapturedE(errors.WithStack(err)))
	}

	defer func() {
		if err := reader.Close(); err != nil {
			logger.Fatal(ctx, "could not close reader", logger.CapturedE(errors.WithStack(err)))
		}
	}()

	if _, err := io.ReadAll(reader); err != nil {
		logger.Fatal(ctx, "could not read blob", logger.CapturedE(errors.WithStack(err)))
	}
}

func writeRandomBlob(ctx context.Context, bucket storage.BlobBucket) {
	blobID := storage.NewBlobID()
	buff := make([]byte, 10*1024)

	writer, err := bucket.NewWriter(ctx, blobID)
	if err != nil {
		logger.Fatal(ctx, "could not create writer", logger.CapturedE(errors.WithStack(err)))
	}

	defer func() {
		if err := writer.Close(); err != nil {
			logger.Fatal(ctx, "could not close writer", logger.CapturedE(errors.WithStack(err)))
		}
	}()

	if _, err := rand.Read(buff); err != nil {
		logger.Fatal(ctx, "could not read random data", logger.CapturedE(errors.WithStack(err)))
	}

	if _, err := writer.Write(buff); err != nil {
		logger.Fatal(ctx, "could not write blob", logger.CapturedE(errors.WithStack(err)))
	}

	printMemUsage(ctx)
}

func printMemUsage(ctx context.Context) {
	var m runtime.MemStats
	runtime.ReadMemStats(&m)
	logger.Debug(
		ctx, "memory usage",
		logger.F("alloc", m.Alloc/1024/1024),
		logger.F("totalAlloc", m.TotalAlloc/1024/1024),
		logger.F("sys", m.Sys/1024/1024),
		logger.F("numGC", m.NumGC),
	)
}