diff --git a/gpu/assets.go b/gpu/assets.go
index 41d0046a..587ddc05 100644
--- a/gpu/assets.go
+++ b/gpu/assets.go
@@ -7,8 +7,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-
-	"github.com/jmorganca/ollama/version"
 )
 
 func AssetsDir() (string, error) {
@@ -16,25 +14,8 @@ func AssetsDir() (string, error) {
 	if err != nil {
 		return "", err
 	}
-	baseDir := filepath.Join(home, ".ollama", "assets")
-	libDirs, err := os.ReadDir(baseDir)
-	if err == nil {
-		for _, d := range libDirs {
-			if d.Name() == version.Version {
-				continue
-			}
-			// Special case the rocm dependencies, which are handled by the installer
-			if d.Name() == "rocm" {
-				continue
-			}
-			slog.Debug("stale lib detected, cleaning up " + d.Name())
-			err = os.RemoveAll(filepath.Join(baseDir, d.Name()))
-			if err != nil {
-				slog.Warn(fmt.Sprintf("unable to clean up stale library %s: %s", filepath.Join(baseDir, d.Name()), err))
-			}
-		}
-	}
-	return filepath.Join(baseDir, version.Version), nil
+
+	return filepath.Join(home, ".ollama", "assets"), nil
 }
 
 func UpdatePath(dir string) {
diff --git a/llm/llm.go b/llm/llm.go
index b0ac0f60..6b5bf961 100644
--- a/llm/llm.go
+++ b/llm/llm.go
@@ -143,25 +143,15 @@ func newLlmServer(gpuInfo gpu.GpuInfo, model string, adapters, projectors []stri
 		}
 	}
 
-	// We stage into a temp directory, and if we've been idle for a while, it may have been reaped
-	_, err := os.Stat(dynLibs[0])
-	if err != nil {
-		slog.Info(fmt.Sprintf("%s has disappeared, reloading libraries", dynLibs[0]))
-		err = nativeInit()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	err2 := fmt.Errorf("unable to locate suitable llm library")
+	err := fmt.Errorf("unable to locate suitable llm library")
 	for _, dynLib := range dynLibs {
 		srv, err := newDynExtServer(dynLib, model, adapters, projectors, opts)
 		if err == nil {
 			return srv, nil
 		}
 		slog.Warn(fmt.Sprintf("Failed to load dynamic library %s  %s", dynLib, err))
-		err2 = err
+		err = err
 	}
 
-	return nil, err2
+	return nil, err
 }
diff --git a/llm/payload_common.go b/llm/payload_common.go
index ff38b63f..791b9d6a 100644
--- a/llm/payload_common.go
+++ b/llm/payload_common.go
@@ -109,6 +109,12 @@ func nativeInit() error {
 	if err != nil {
 		return err
 	}
+
+	// delete the assetsDir
+	if err := os.RemoveAll(assetsDir); err != nil {
+		return err
+	}
+
 	if runtime.GOOS == "darwin" {
 		err := extractPayloadFiles(assetsDir, "llama.cpp/ggml-metal.metal")
 		if err != nil {