diff --git a/core/cache/file_caches.go b/core/cache/file_caches.go index 6ea583f02..0e05d97d4 100644 --- a/core/cache/file_caches.go +++ b/core/cache/file_caches.go @@ -201,5 +201,14 @@ func newFSCache(name, cacheSize, cacheFolder string, maxItems int) (fscache.Cach return nil, err } - return fscache.NewCacheWithHaunter(fs, h) + ck, err := fscache.NewCacheWithHaunter(fs, h) + if err != nil { + return nil, err + } + + if conf.Server.DevNewCacheLayout { + ck.SetKeyMapper(fs.(*spreadFS).KeyMapper) + } + + return ck, nil } diff --git a/core/cache/spread_fs.go b/core/cache/spread_fs.go index 813ca79f7..13f28e513 100644 --- a/core/cache/spread_fs.go +++ b/core/cache/spread_fs.go @@ -3,7 +3,6 @@ package cache import ( "crypto/sha1" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -20,8 +19,6 @@ type spreadFS struct { init func() error } -const keyFileExtension = ".key" - // NewSpreadFS returns a FileSystem rooted at directory dir. It // Dir is created with perms if it doesn't exist. func NewSpreadFS(dir string, mode os.FileMode) (fscache.FileSystem, error) { @@ -39,32 +36,13 @@ func (fs *spreadFS) Reload(f func(key string, name string)) error { return nil } - // Skip if name is not in the format XX/XX/XXXXXXXXXXXX.key + // Skip if name is not in the format XX/XX/XXXXXXXXXXXX parts := strings.Split(path, string(os.PathSeparator)) - if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 || - filepath.Ext(path) != keyFileExtension { + if len(parts) != 3 || len(parts[0]) != 2 || len(parts[1]) != 2 { return nil } - keyFileName := absoluteFilePath - dataFileName := absoluteFilePath[0 : len(absoluteFilePath)-len(keyFileExtension)] - - // Load the key from the key file. Remove and skip on error - key, err := ioutil.ReadFile(keyFileName) - if err != nil { - _ = fs.Remove(dataFileName) - return nil - } - - // If the data file is not readable, remove and skip - file, err := os.Open(dataFileName) - defer func() { _ = file.Close() }() - if err != nil { - _ = fs.Remove(dataFileName) - return nil - } - - f(string(key), dataFileName) + f(absoluteFilePath, absoluteFilePath) return nil }, Unsorted: true, @@ -72,18 +50,12 @@ func (fs *spreadFS) Reload(f func(key string, name string)) error { } func (fs *spreadFS) Create(name string) (stream.File, error) { - key := fmt.Sprintf("%x", sha1.Sum([]byte(name))) - path := fmt.Sprintf("%s%c%s", key[0:2], os.PathSeparator, key[2:4]) - err := os.MkdirAll(filepath.Join(fs.root, path), fs.mode) + path := filepath.Dir(name) + err := os.MkdirAll(path, fs.mode) if err != nil { return nil, err } - absolutePath := filepath.Join(fs.root, path, key) - err = ioutil.WriteFile(absolutePath+keyFileExtension, []byte(name), 0600) - if err != nil { - return nil, err - } - return os.OpenFile(absolutePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + return os.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) } func (fs *spreadFS) Open(name string) (stream.File, error) { @@ -91,7 +63,6 @@ func (fs *spreadFS) Open(name string) (stream.File, error) { } func (fs *spreadFS) Remove(name string) error { - _ = os.Remove(name + keyFileExtension) return os.Remove(name) } @@ -109,3 +80,8 @@ func (fs *spreadFS) RemoveAll() error { } return fs.init() } + +func (fs *spreadFS) KeyMapper(key string) string { + hash := fmt.Sprintf("%x", sha1.Sum([]byte(key))) + return filepath.Join(fs.root, hash[0:2], hash[2:4], hash) +} diff --git a/go.mod b/go.mod index 17d4c9b2d..e34bd73b5 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/dhowden/tag v0.0.0-20200412032933-5d76b8eaae27 github.com/disintegration/imaging v1.6.2 - github.com/djherbis/fscache v0.10.1 + github.com/djherbis/fscache v0.10.2-0.20201024185917-a0daa9e52747 github.com/dustin/go-humanize v1.0.0 github.com/go-bindata/go-bindata v3.1.2+incompatible github.com/go-chi/chi v4.1.2+incompatible diff --git a/go.sum b/go.sum index 3fa36b959..187ef9fe7 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/fscache v0.10.1 h1:hDv+RGyvD+UDKyRYuLoVNbuRTnf2SrA2K3VyR1br9lk= github.com/djherbis/fscache v0.10.1/go.mod h1:yyPYtkNnnPXsW+81lAcQS6yab3G2CRfnPLotBvtbf0c= +github.com/djherbis/fscache v0.10.2-0.20201024185917-a0daa9e52747 h1:bA5vjwEwlH3A5zJON9SIUcpcIH5uE42rfRICa+O5JTw= +github.com/djherbis/fscache v0.10.2-0.20201024185917-a0daa9e52747/go.mod h1:yyPYtkNnnPXsW+81lAcQS6yab3G2CRfnPLotBvtbf0c= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 h1:aaQcKT9WumO6JEJcRyTqFVq4XUZiUcKR2/GI31TOcz8=