diff --git a/server/images.go b/server/images.go
index 96966f6f..222b0a39 100644
--- a/server/images.go
+++ b/server/images.go
@@ -602,139 +602,6 @@ func CopyModel(src, dst model.Name) error {
 	return err
 }
 
-func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}) error {
-	fp, err := GetManifestPath()
-	if err != nil {
-		return err
-	}
-
-	walkFunc := func(path string, info os.FileInfo, _ error) error {
-		if info.IsDir() {
-			return nil
-		}
-
-		dir, file := filepath.Split(path)
-		dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
-		tag := strings.Join([]string{dir, file}, ":")
-		fmp := ParseModelPath(tag)
-
-		// skip the manifest we're trying to delete
-		if skipModelPath != nil && skipModelPath.GetFullTagname() == fmp.GetFullTagname() {
-			return nil
-		}
-
-		// save (i.e. delete from the deleteMap) any files used in other manifests
-		manifest, _, err := GetManifest(fmp)
-		if err != nil {
-			//nolint:nilerr
-			return nil
-		}
-
-		for _, layer := range manifest.Layers {
-			delete(deleteMap, layer.Digest)
-		}
-
-		delete(deleteMap, manifest.Config.Digest)
-		return nil
-	}
-
-	if err := filepath.Walk(fp, walkFunc); err != nil {
-		return err
-	}
-
-	// only delete the files which are still in the deleteMap
-	for k := range deleteMap {
-		fp, err := GetBlobsPath(k)
-		if err != nil {
-			slog.Info(fmt.Sprintf("couldn't get file path for '%s': %v", k, err))
-			continue
-		}
-		if err := os.Remove(fp); err != nil {
-			slog.Info(fmt.Sprintf("couldn't remove file '%s': %v", fp, err))
-			continue
-		}
-	}
-
-	return nil
-}
-
-func PruneLayers() error {
-	deleteMap := make(map[string]struct{})
-	p, err := GetBlobsPath("")
-	if err != nil {
-		return err
-	}
-
-	blobs, err := os.ReadDir(p)
-	if err != nil {
-		slog.Info(fmt.Sprintf("couldn't read dir '%s': %v", p, err))
-		return err
-	}
-
-	for _, blob := range blobs {
-		name := blob.Name()
-		name = strings.ReplaceAll(name, "-", ":")
-
-		_, err := GetBlobsPath(name)
-		if err != nil {
-			if errors.Is(err, ErrInvalidDigestFormat) {
-				// remove invalid blobs (e.g. partial downloads)
-				if err := os.Remove(filepath.Join(p, blob.Name())); err != nil {
-					slog.Error("couldn't remove blob", "blob", blob.Name(), "error", err)
-				}
-			}
-
-			continue
-		}
-
-		deleteMap[name] = struct{}{}
-	}
-
-	slog.Info(fmt.Sprintf("total blobs: %d", len(deleteMap)))
-
-	err = deleteUnusedLayers(nil, deleteMap)
-	if err != nil {
-		return err
-	}
-
-	slog.Info(fmt.Sprintf("total unused blobs removed: %d", len(deleteMap)))
-
-	return nil
-}
-
-func PruneDirectory(path string) error {
-	info, err := os.Lstat(path)
-	if err != nil {
-		return err
-	}
-
-	if info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
-		entries, err := os.ReadDir(path)
-		if err != nil {
-			return err
-		}
-
-		for _, entry := range entries {
-			if err := PruneDirectory(filepath.Join(path, entry.Name())); err != nil {
-				return err
-			}
-		}
-
-		entries, err = os.ReadDir(path)
-		if err != nil {
-			return err
-		}
-
-		if len(entries) > 0 {
-			return nil
-		}
-
-		return os.Remove(path)
-	}
-
-	return nil
-}
-
 func PushModel(ctx context.Context, name string, regOpts *registryOptions, fn func(api.ProgressResponse)) error {
 	mp := ParseModelPath(name)
 	fn(api.ProgressResponse{Status: "retrieving manifest"})
diff --git a/server/layer.go b/server/layer.go
index d3d3c120..ad8c2069 100644
--- a/server/layer.go
+++ b/server/layer.go
@@ -4,7 +4,10 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"log/slog"
 	"os"
+	"path/filepath"
+	"strings"
 )
 
 type Layer struct {
@@ -104,10 +107,45 @@ func (l *Layer) Remove() error {
 		}
 	}
 
-	blob, err := GetBlobsPath(l.Digest)
+	p, err := GetBlobsPath("")
 	if err != nil {
 		return err
 	}
 
-	return os.Remove(blob)
+	return os.Remove(filepath.Join(p, l.Digest))
+}
+
+func Layers() (map[string]*Layer, error) {
+	blobs, err := GetBlobsPath("")
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO(mxyng): use something less brittle
+	matches, err := filepath.Glob(filepath.Join(blobs, "*"))
+	if err != nil {
+		return nil, err
+	}
+
+	ds := make(map[string]*Layer)
+	for _, match := range matches {
+		rel, err := filepath.Rel(blobs, match)
+		if err != nil {
+			slog.Warn("bad filepath", "path", match, "error", err)
+			continue
+		}
+
+		// TODO(mxyng): this should ideally use model.Digest but
+		// that's currently incompatible with the manifest digest
+		d := strings.Replace(rel, "sha256-", "sha256:", 1)
+		layer, err := NewLayerFromLayer(d, "", "")
+		if err != nil {
+			slog.Warn("bad blob", "digest", d, "error", err)
+			layer = &Layer{Digest: rel}
+		}
+
+		ds[d] = layer
+	}
+
+	return ds, nil
 }
diff --git a/server/manifest.go b/server/manifest.go
index d0675724..2856db9b 100644
--- a/server/manifest.go
+++ b/server/manifest.go
@@ -38,7 +38,42 @@ func (m *Manifest) Remove() error {
 		return err
 	}
 
-	return PruneDirectory(manifests)
+	return pruneEmptyDirectory(manifests)
+}
+
+func pruneEmptyDirectory(p string) error {
+	fi, err := os.Lstat(p)
+	if err != nil {
+		return err
+	}
+
+	if fi.Mode()&os.ModeSymlink == 0 {
+		entries, err := os.ReadDir(p)
+		if err != nil {
+			return err
+		}
+
+		for _, entry := range entries {
+			if entry.IsDir() {
+				if err := pruneEmptyDirectory(filepath.Join(p, entry.Name())); err != nil {
+					return err
+				}
+			}
+		}
+
+		entries, err = os.ReadDir(p)
+		if err != nil {
+			return err
+		}
+
+		if len(entries) == 0 {
+			if err := os.Remove(p); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
 }
 
 func (m *Manifest) RemoveLayers() error {
diff --git a/server/routes.go b/server/routes.go
index 55c49970..7c35cef4 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -973,17 +973,15 @@ func Serve(ln net.Listener) error {
 
 	if !envconfig.NoPrune {
 		// clean up unused layers and manifests
-		if err := PruneLayers(); err != nil {
-			return err
-		}
-
-		manifestsPath, err := GetManifestPath()
+		layers, err := Layers()
 		if err != nil {
 			return err
 		}
 
-		if err := PruneDirectory(manifestsPath); err != nil {
-			return err
+		for _, layer := range layers {
+			if err := layer.Remove(); err != nil {
+				return err
+			}
 		}
 	}