Use shardedfilestore for image-proxy

Author: ItsOnlyBinary

server/embed.go

@@ -21,8 +21,6 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/dyatlov/go-oembed/oembed"
 	"github.com/gin-gonic/gin"
-	"github.com/gregjones/httpcache/diskcache"
-	"github.com/peterbourgon/diskv"
 	"willnorris.com/go/imageproxy"
 )
 
@@ -72,7 +70,7 @@ func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error
 	rg.GET("", handleEmbed)
 
 	// Attach imageproxy
-	cache := diskCache(cfg.Embed.ImageCachePath, cfg.Embed.ImageCacheMaxSize)
+	cache := serv.diskCache()
 	imgProxy = imageproxy.NewProxy(nil, cache)
 
 	ic := r.Group("/image-cache/*id")
@@ -83,18 +81,20 @@ func (serv *UploadServer) registerEmbedHandlers(r *gin.Engine, cfg Config) error
 func handleImageCache(c *gin.Context) {
 	r := c.Request
 	r.URL.Path = strings.Replace(r.URL.Path, "/image-cache", "", -1)
-	spew.Dump(r.URL.Path)
 	imgProxy.ServeHTTP(c.Writer, c.Request)
+	fmt.Println("------------------------------------------------------")
 }
 
-func diskCache(path string, maxSize uint64) *diskcache.Cache {
-	d := diskv.New(diskv.Options{
-		BasePath:     path,
-		CacheSizeMax: maxSize,
-		// For file "c0ffee", store file as "c0/ff/c0ffee"
-		Transform: func(s string) []string { return []string{s[0:2], s[2:4]} },
-	})
-	return diskcache.NewWithDiskv(d)
+func (serv *UploadServer) diskCache() *ImageProxyCache {
+	// d := diskv.New(diskv.Options{
+	// 	BasePath:     path,
+	// 	CacheSizeMax: maxSize,
+	// 	// For file "c0ffee", store file as "c0/ff/c0ffee"
+	// 	Transform: func(s string) []string { return []string{s[0:2], s[2:4]} },
+	// })
+	// return diskcache.NewWithDiskv(d)
+	d := NewImageProxyCache(serv.store, serv.log)
+	return d
 }
 
 func handleEmbed(c *gin.Context) {

server/image-proxy-cache.go

@@ -0,0 +1,111 @@
+package server
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/kiwiirc/plugin-fileuploader/shardedfilestore"
+	"github.com/rs/zerolog"
+	"github.com/tus/tusd"
+)
+
+// ImageProxyCache is an implementation of httpcache.Cache that supplements the in-memory map with persistent storage
+type ImageProxyCache struct {
+	store  *shardedfilestore.ShardedFileStore
+	log    *zerolog.Logger
+	urlMap sync.Map
+}
+
+// Get returns the response corresponding to key if present
+func (c *ImageProxyCache) Get(key string) (resp []byte, ok bool) {
+	urlHash := getHash(key)
+	spew.Dump("Get", key, urlHash)
+
+	idInterface, ok := c.urlMap.Load(urlHash)
+	if !ok {
+		fmt.Println("Not in map")
+		fmt.Println("")
+		return []byte{}, false
+	}
+	id := idInterface.(string)
+
+	reader, err := c.store.GetReader(id)
+	if err != nil {
+		fmt.Println("No reader")
+		fmt.Println("")
+		c.urlMap.Delete(urlHash)
+		return []byte{}, false
+	}
+
+	buffer := new(bytes.Buffer)
+	_, err = buffer.ReadFrom(reader)
+	if err != nil {
+		fmt.Println("Failed read")
+		fmt.Println("")
+		c.urlMap.Delete(urlHash)
+		return []byte{}, false
+	}
+
+	bytes := buffer.Bytes()
+
+	spew.Dump(len(bytes))
+	fmt.Println("Got from cache")
+	fmt.Println("")
+	return bytes, true
+}
+
+// Set saves a response to the cache as key
+func (c *ImageProxyCache) Set(key string, resp []byte) {
+	urlHash := getHash(key)
+	spew.Dump("Set", key, len(resp), urlHash)
+
+	metaData := tusd.MetaData{
+		"Url": key,
+	}
+	fileInfo := tusd.FileInfo{
+		Size:           int64(len(resp)),
+		SizeIsDeferred: false,
+		MetaData:       metaData,
+		IsFinal:        false,
+	}
+
+	id, err := c.store.NewUpload(fileInfo)
+	if err != nil {
+		c.log.Error().
+			Err(err).
+			Msg("Failed to create new upload")
+		return
+	}
+
+	_, err = c.store.WriteChunk(id, 0, bytes.NewReader(resp))
+	if err != nil {
+		c.log.Error().
+			Err(err).
+			Msg("Failed to write chunk")
+		return
+	}
+
+	c.store.FinishUpload(id)
+	c.urlMap.Store(urlHash, id)
+
+	fmt.Println("Set in cache")
+	fmt.Println("")
+}
+
+// Delete removes the response with key from the cache
+func (c *ImageProxyCache) Delete(key string) {
+	urlHash := getHash(key)
+	c.urlMap.Delete(urlHash)
+	spew.Dump("Delete", key)
+	fmt.Println("")
+}
+
+// NewImageProxyCache returns a new Cache that will store files in basePath
+func NewImageProxyCache(store *shardedfilestore.ShardedFileStore, log *zerolog.Logger) *ImageProxyCache {
+	return &ImageProxyCache{
+		store: store,
+		log:   log,
+	}
+}

server/uploadserver.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"context"
 	"net/http"
 	"sync"
 
@@ -18,6 +19,7 @@ import (
 type UploadServer struct {
 	DBConn *db.DatabaseConnection
 	Router *gin.Engine
+	ctx    *RunContext
 
 	cfg                 Config
 	log                 *zerolog.Logger
@@ -109,7 +111,7 @@ func (serv *UploadServer) Shutdown() {
 
 	// wait for all requests to finish
 	if serv.httpServer != nil {
-		serv.httpServer.Shutdown(nil)
+		serv.httpServer.Shutdown(context.TODO())
 	}
 
 	// stop running FileStore GC cycles