Fix cache key-mapping

.golangci.yml

@@ -26,4 +26,4 @@ issues:
   exclude-rules:
     - linters:
         - gosec
-      text: "(G501|G401):"
+      text: "(G501|G401|G505):"

core/cache/file_caches.go

@@ -184,7 +184,7 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach
 	cacheFolder = filepath.Join(conf.Server.DataFolder, cacheFolder)
 
 	log.Info(fmt.Sprintf("Creating %s cache", name), "path", cacheFolder, "maxSize", humanize.Bytes(size))
-	fs, err := NewSpreadFs(cacheFolder, 0755)
+	fs, err := NewSpreadFS(cacheFolder, 0755)
 	if err != nil {
 		log.Error(fmt.Sprintf("Error initializing %s cache", name), err, "elapsedTime", time.Since(start))
 		return nil, err

core/cache/spread_fs.go

@@ -1,8 +1,9 @@
 package cache
 
 import (
-	"crypto/md5"
+	"crypto/sha1"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -13,22 +14,24 @@ import (
 	"gopkg.in/djherbis/stream.v1"
 )
 
-type spreadFs struct {
+type spreadFS struct {
 	root string
 	mode os.FileMode
 	init func() error
 }
 
-// NewSpreadFs returns a FileSystem rooted at directory dir. It
+const keyFileExtension = ".key"
+
+// NewSpreadFS returns a FileSystem rooted at directory dir. It
 // Dir is created with perms if it doesn't exist.
-func NewSpreadFs(dir string, mode os.FileMode) (fscache.FileSystem, error) {
-	fs := &spreadFs{root: dir, mode: mode, init: func() error {
+func NewSpreadFS(dir string, mode os.FileMode) (fscache.FileSystem, error) {
+	fs := &spreadFS{root: dir, mode: mode, init: func() error {
 		return os.MkdirAll(dir, mode)
 	}}
 	return fs, fs.init()
 }
 
-func (fs *spreadFs) Reload(f func(key string, name string)) error {
+func (fs *spreadFS) Reload(f func(key string, name string)) error {
 	return godirwalk.Walk(fs.root, &godirwalk.Options{
 		Callback: func(absoluteFilePath string, de *godirwalk.Dirent) error {
 			path, err := filepath.Rel(fs.root, absoluteFilePath)
@@ -36,38 +39,63 @@ func (fs *spreadFs) Reload(f func(key string, name string)) error {
 				return nil
 			}
 
+			// Skip if name is not in the format XX/XX/XXXXXXXXXXXX.key
 			parts := strings.Split(path, string(os.PathSeparator))
-			if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 {
+			if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 ||
+				filepath.Ext(path) != keyFileExtension {
 				return nil
 			}
 
-			key := filepath.Base(path)
-			f(key, absoluteFilePath)
+			keyFileName := absoluteFilePath
+			dataFileName := absoluteFilePath[0 : len(absoluteFilePath)-len(keyFileExtension)]
+
+			// Load the key from the key file. Remove and skip on error
+			key, err := ioutil.ReadFile(keyFileName)
+			if err != nil {
+				_ = fs.Remove(dataFileName)
+				return nil
+			}
+
+			// If the data file is not readable, remove and skip
+			file, err := os.Open(dataFileName)
+			defer func() { _ = file.Close() }()
+			if err != nil {
+				_ = fs.Remove(dataFileName)
+				return nil
+			}
+
+			f(string(key), dataFileName)
 			return nil
 		},
 		Unsorted: true,
 	})
 }
 
-func (fs *spreadFs) Create(name string) (stream.File, error) {
-	key := fmt.Sprintf("%x", md5.Sum([]byte(name)))
+func (fs *spreadFS) Create(name string) (stream.File, error) {
+	key := fmt.Sprintf("%x", sha1.Sum([]byte(name)))
 	path := fmt.Sprintf("%s%c%s", key[0:2], os.PathSeparator, key[2:4])
 	err := os.MkdirAll(filepath.Join(fs.root, path), fs.mode)
 	if err != nil {
 		return nil, err
 	}
-	return os.OpenFile(filepath.Join(fs.root, path, key), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	absolutePath := filepath.Join(fs.root, path, key)
+	err = ioutil.WriteFile(absolutePath+keyFileExtension, []byte(name), 0600)
+	if err != nil {
+		return nil, err
+	}
+	return os.OpenFile(absolutePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 }
 
-func (fs *spreadFs) Open(name string) (stream.File, error) {
+func (fs *spreadFS) Open(name string) (stream.File, error) {
 	return os.Open(name)
 }
 
-func (fs *spreadFs) Remove(name string) error {
+func (fs *spreadFS) Remove(name string) error {
+	_ = os.Remove(name + keyFileExtension)
 	return os.Remove(name)
 }
 
-func (fs *spreadFs) Stat(name string) (fscache.FileInfo, error) {
+func (fs *spreadFS) Stat(name string) (fscache.FileInfo, error) {
 	stat, err := os.Stat(name)
 	if err != nil {
 		return fscache.FileInfo{}, err
@@ -75,7 +103,7 @@ func (fs *spreadFs) Stat(name string) (fscache.FileInfo, error) {
 	return fscache.FileInfo{FileInfo: stat, Atime: atime.Get(stat)}, nil
 }
 
-func (fs *spreadFs) RemoveAll() error {
+func (fs *spreadFS) RemoveAll() error {
 	if err := os.RemoveAll(fs.root); err != nil {
 		return err
 	}