diff --git a/Dockerfile b/Dockerfile
index cdb756c0..59cca725 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,11 +20,11 @@ WORKDIR /go/src/github.com/jmorganca/ollama/llm/generate
 ARG CGO_CFLAGS
 RUN OLLAMA_SKIP_CPU_GENERATE=1 sh gen_linux.sh
 
-FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION-devel-centos7 AS cuda-build-arm64
+FROM --platform=linux/arm64 nvidia/cuda:$CUDA_VERSION-devel-rockylinux8 AS cuda-build-arm64
 ARG CMAKE_VERSION
 COPY ./scripts/rh_linux_deps.sh /
 RUN CMAKE_VERSION=${CMAKE_VERSION} sh /rh_linux_deps.sh
-ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
+ENV PATH /opt/rh/gcc-toolset-10/root/usr/bin:$PATH
 COPY --from=llm-code / /go/src/github.com/jmorganca/ollama/
 WORKDIR /go/src/github.com/jmorganca/ollama/llm/generate
 ARG CGO_CFLAGS
diff --git a/scripts/rh_linux_deps.sh b/scripts/rh_linux_deps.sh
index ec6b20a0..ed60e430 100644
--- a/scripts/rh_linux_deps.sh
+++ b/scripts/rh_linux_deps.sh
@@ -20,7 +20,19 @@ if grep -i "centos" /etc/system-release >/dev/null; then
     fi
     dnf install -y devtoolset-10-gcc devtoolset-10-gcc-c++
 elif grep -i "rocky" /etc/system-release >/dev/null; then
-    dnf install -y git gcc-toolset-10-gcc gcc-toolset-10-gcc-c++
+    # Temporary workaround until rocky 8 AppStream ships GCC 10.4 (10.3 is incompatible with NVCC)
+    cat << EOF > /etc/yum.repos.d/Rocky-Vault.repo
+[vault]
+name=Rocky Vault
+baseurl=https://dl.rockylinux.org/vault/rocky/8.5/AppStream/\$basearch/os/
+gpgcheck=1
+enabled=1
+countme=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial
+EOF
+    dnf install -y git \
+        gcc-toolset-10-gcc-10.2.1-8.2.el8 \
+        gcc-toolset-10-gcc-c++-10.2.1-8.2.el8
 else
     echo "ERROR Unexpected distro"
     exit 1