diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 8ca96865..5c3c4600 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -170,20 +170,16 @@ jobs:
       - name: 'gather rocm dependencies'
         run: |
           $HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
-          md "dist\deps\bin\rocblas\library"
-          cp "${HIP_PATH}\bin\hipblas.dll" "dist\deps\bin\"
-          cp "${HIP_PATH}\bin\rocblas.dll" "dist\deps\bin\"
-          cp "${HIP_PATH}\bin\rocblas\library\*" "dist\deps\bin\rocblas\library\"
+          md "dist\windows-amd64\lib\ollama\rocblas\library"
+          cp "${HIP_PATH}\bin\hipblas.dll" "dist\windows-amd64\lib\ollama\"
+          cp "${HIP_PATH}\bin\rocblas.dll" "dist\windows-amd64\lib\ollama\"
+          cp "${HIP_PATH}\bin\rocblas\library\*" "dist\windows-amd64\lib\ollama\rocblas\library\"
       - uses: actions/upload-artifact@v4
         with:
           name: generate-windows-rocm
           path: |
             build/**/*
             dist/windows-amd64/**
-      - uses: actions/upload-artifact@v4
-        with:
-          name: windows-rocm-deps
-          path: dist/deps/*
 
   # CUDA generation step
   generate-windows-cuda:
@@ -260,21 +256,15 @@ jobs:
       - name: 'gather cuda dependencies'
         run: |
           $NVIDIA_DIR=(resolve-path 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*\bin\')[0]
-          md "dist\deps"
-          cp "${NVIDIA_DIR}\cudart64_*.dll" "dist\deps\"
-          cp "${NVIDIA_DIR}\cublas64_*.dll" "dist\deps\"
-          cp "${NVIDIA_DIR}\cublasLt64_*.dll" "dist\deps\"
+          cp "${NVIDIA_DIR}\cudart64_*.dll" "dist\windows-amd64\lib\ollama\"
+          cp "${NVIDIA_DIR}\cublas64_*.dll" "dist\windows-amd64\lib\ollama\"
+          cp "${NVIDIA_DIR}\cublasLt64_*.dll" "dist\windows-amd64\lib\ollama\"
       - uses: actions/upload-artifact@v4
         with:
           name: generate-windows-cuda-${{ matrix.cuda.version }}
           path: |
             build/**/*
             dist/windows-amd64/**
-      - uses: actions/upload-artifact@v4
-        with:
-          name: windows-cuda-deps-${{ matrix.cuda.version }}
-          path: dist/deps/*
-
 
   # windows arm64 generate, go build, and zip file (no installer)
   # Output of this build is aggregated into the final x86 build
@@ -480,15 +470,6 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: generate-windows-cuda-12
-      - uses: actions/download-artifact@v4
-        with:
-          name: windows-cuda-deps-11
-      - uses: actions/download-artifact@v4
-        with:
-          name: windows-cuda-deps-12
-      - uses: actions/download-artifact@v4
-        with:
-          name: windows-rocm-deps
       - uses: actions/download-artifact@v4
         with:
           name: generate-windows-rocm
diff --git a/scripts/build_windows.ps1 b/scripts/build_windows.ps1
index 670b39b3..3ac33987 100644
--- a/scripts/build_windows.ps1
+++ b/scripts/build_windows.ps1
@@ -143,6 +143,22 @@ function gatherDependencies() {
         copy-item -path "${env:VCToolsRedistDir}\vc_redist.arm64.exe" -destination "${script:DIST_DIR}" -verbose
     }
 
+    foreach ($script:CUDA_LIB_DIR in $script:CUDA_DIRS) {
+        write-host "copying CUDA dependencies to ${script:DIST_DIR}\lib\ollama\"
+        cp "${script:CUDA_LIB_DIR}\cudart64_*.dll" "${script:DIST_DIR}\lib\ollama\"
+        cp "${script:CUDA_LIB_DIR}\cublas64_*.dll" "${script:DIST_DIR}\lib\ollama\"
+        cp "${script:CUDA_LIB_DIR}\cublasLt64_*.dll" "${script:DIST_DIR}\lib\ollama\"
+    }
+
+    if ($null -ne ${env:HIP_PATH}) {
+        md "${script:DIST_DIR}\lib\ollama\rocblas\library\" -ea 0 > $null
+        cp "${env:HIP_PATH}\bin\hipblas.dll" "${script:DIST_DIR}\lib\ollama\"
+        cp "${env:HIP_PATH}\bin\rocblas.dll" "${script:DIST_DIR}\lib\ollama\"
+        # amdhip64.dll dependency comes from the driver and must be installed on the host to use AMD GPUs
+        cp "${env:HIP_PATH}\bin\rocblas\library\*" "${script:DIST_DIR}\lib\ollama\rocblas\library\"
+    }
+
+    # TODO - oneapi
 
     cp "${script:SRC_DIR}\app\ollama_welcome.ps1" "${script:SRC_DIR}\dist\"
     if ("${env:KEY_CONTAINER}") {