read mem requirements

The build tags rocm or cuda must be specified to both go generate and go build.
ROCm builds should have both ROCM_PATH set (and the ROCM SDK present) as well
as CLBlast installed (for GGML) and CLBlast_DIR set in the environment to the
CLBlast cmake directory (likely /usr/lib/cmake/CLBlast). Build tags are also
used to switch VRAM detection between cuda and rocm implementations, using
added "accelerator_foo.go" files which contain architecture specific functions
and variables. accelerator_none is used when no tags are set, and a helper
function addRunner will ignore it if it is the chosen accelerator. Fix go
generate commands, thanks @deadmeu for testing.

.dockerignore

@@ -2,8 +2,7 @@
 ollama
 app
 dist
-scripts
-llm/llama.cpp/ggml
 llm/llama.cpp/gguf
 .env
 .cache
+test_data

.gitignore

@@ -8,4 +8,5 @@ ollama
 ggml-metal.metal
 .cache
 *.exe
-.idea
+.idea
+test_data

.gitmodules

@@ -1,8 +1,3 @@
-[submodule "llm/llama.cpp/ggml"]
-    path = llm/llama.cpp/ggml
-    url = https://github.com/ggerganov/llama.cpp.git
-    ignore = dirty
-    shallow = true
 [submodule "llm/llama.cpp/gguf"]
     path = llm/llama.cpp/gguf
     url = https://github.com/ggerganov/llama.cpp.git

Dockerfile

@@ -11,8 +11,8 @@ RUN mkdir -p /usr/local && tar xz -C /usr/local </tmp/go1.21.3.tar.gz
 COPY . .
 ENV GOARCH=$TARGETARCH
 ENV GOFLAGS=$GOFLAGS
-RUN /usr/local/go/bin/go generate ./... \
-    && /usr/local/go/bin/go build .
+RUN /usr/local/go/bin/go generate -tags cuda ./... \
+    && /usr/local/go/bin/go build -tags cuda .
 
 FROM ubuntu:22.04
 RUN apt-get update && apt-get install -y ca-certificates
@@ -27,3 +27,5 @@ ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 
 ENTRYPOINT ["/bin/ollama"]
 CMD ["serve"]
+
+

Dockerfile.build

@@ -1,19 +1,45 @@
-# centos7 amd64 dependencies
-FROM --platform=linux/amd64 nvidia/cuda:11.3.1-devel-centos7 AS base-amd64
-RUN yum install -y https://repo.ius.io/ius-release-el7.rpm centos-release-scl && \
-    yum update -y && \
-    yum install -y devtoolset-10-gcc devtoolset-10-gcc-c++ git236 wget
-RUN wget "https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-x86_64.sh" -O cmake-installer.sh && chmod +x cmake-installer.sh && ./cmake-installer.sh --skip-license --prefix=/usr/local
-ENV PATH /opt/rh/devtoolset-10/root/usr/bin:$PATH
+# Ubuntu 20.04 amd64 dependencies
+FROM --platform=linux/amd64 nvidia/cuda:11.7.1-devel-ubuntu22.04 AS base-amd64
+# ROCm only supports amd64
+ARG ROCM_VERSION=5.7
+# Note: https://rocm.docs.amd.com/en/latest/release/user_kernel_space_compat_matrix.html
+RUN apt-get update && \
+    apt-get install -y wget && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.22.1/cmake-3.22.1-linux-x86_64.sh" -O /tmp/cmake-installer.sh && \
+    chmod +x /tmp/cmake-installer.sh && /tmp/cmake-installer.sh --skip-license --prefix=/usr && \
+    mkdir --parents --mode=0755 /etc/apt/keyrings && \
+    wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \
+    echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/${ROCM_VERSION} focal main" > /etc/apt/sources.list.d/rocm.list && \
+    echo "Package: *" > /etc/apt/preferences.d/rocm-pin-600 && \
+    echo "Pin: release o=repo.radeon.com" >> /etc/apt/preferences.d/rocm-pin-600 && \
+    echo "Pin-Priority: 600" >> /etc/apt/preferences.d/rocm-pin-600 && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y rocm-hip-libraries rocm-device-libs rocm-libs rocm-ocl-icd rocm-hip-sdk rocm-hip-libraries rocm-cmake rocm-clang-ocl rocm-dev
 
-# centos8 arm64 dependencies
-FROM --platform=linux/arm64 nvidia/cuda-arm64:11.3.1-devel-centos8 AS base-arm64
-RUN sed -i -e 's/mirrorlist/#mirrorlist/g' -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-RUN yum install -y git cmake
+ENV ROCM_PATH=/opt/rocm
+
+# Ubuntu 22.04 arm64 dependencies
+FROM --platform=linux/arm64 nvidia/cuda:11.7.1-devel-ubuntu22.04 AS base-arm64
+RUN apt-get update && \
+    apt-get install -y wget && \
+    wget "https://github.com/Kitware/CMake/releases/download/v3.27.6/cmake-3.27.6-linux-aarch64.sh" -O /tmp/cmake-installer.sh && \
+    chmod +x /tmp/cmake-installer.sh && /tmp/cmake-installer.sh --skip-license --prefix=/usr
 
 FROM base-${TARGETARCH}
 ARG TARGETARCH
 ARG GOFLAGS="'-ldflags -w -s'"
+ARG CGO_CFLAGS
+ARG CLBLAST_VER=1.6.1
+
+# Common toolchain
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-11 g++-11 cpp-11 git ocl-icd-opencl-dev && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 --slave /usr/bin/g++ g++ /usr/bin/g++-11 --slave /usr/bin/gcov gcov /usr/bin/gcov-11
+
+# CLBlast
+RUN wget -qO- https://github.com/CNugteren/CLBlast/archive/refs/tags/${CLBLAST_VER}.tar.gz | tar zxv -C /tmp/ && \
+    cd /tmp/CLBlast-${CLBLAST_VER} && mkdir build && cd build && cmake .. && make && make install
+ENV CLBlast_DIR=/usr/lib/cmake/CLBlast
 
 # install go
 ADD https://dl.google.com/go/go1.21.3.linux-$TARGETARCH.tar.gz /tmp/go1.21.3.tar.gz
@@ -26,6 +52,7 @@ COPY . .
 ENV GOOS=linux
 ENV GOARCH=$TARGETARCH
 ENV GOFLAGS=$GOFLAGS
+ENV CGO_CFLAGS=${CGO_CFLAGS}
 
-RUN /usr/local/go/bin/go generate ./... && \
+RUN /usr/local/go/bin/go generate  ./... && \
     /usr/local/go/bin/go build .

README.md

@@ -177,19 +177,50 @@ ollama list
 
 ## Building
 
+### Generic (CPU)
+
 Install `cmake` and `go`:
 
 ```
 brew install cmake go
 ```
 
-Then generate dependencies and build:
-
+Then generate dependencies:
 ```
 go generate ./...
+```
+Then build the binary:
+```
 go build .
 ```
 
+### CUDA (NVIDIA)
+*Your operating system distribution may already have packages for NVIDIA CUDA. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!*
+
+Install `cmake` and `golang` as well as [NVIDIA CUDA](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html) development and runtime packages.
+Then generate dependencies:
+```
+go generate -tags cuda ./...
+```
+Then build the binary:
+```
+go build -tags cuda .
+```
+
+### ROCm (AMD)
+*Your operating system distribution may already have packages for AMD ROCm and CLBlast. Distro packages are often preferable, but instructions are distro-specific. Please consult distro-specific docs for dependencies if available!*
+
+Install [CLBlast](https://github.com/CNugteren/CLBlast/blob/master/doc/installation.md) and [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/quick_start.html) developement packages first, as well as `cmake` and `golang`.
+Adjust the paths below (correct for Arch) as appropriate for your distributions install locations and generate dependencies:
+```
+CLBlast_DIR=/usr/lib/cmake/CLBlast ROCM_PATH=/opt/rocm go generate -tags rocm ./...
+```
+Then build the binary:
+```
+go build -tags rocm
+```
+
+### Running local builds
 Next, start the server:
 
 ```

cmd/cmd.go

@@ -572,10 +572,30 @@ func generate(cmd *cobra.Command, opts generateOptions) error {
 	}
 
 	if err := client.Generate(ctx, &request, fn); err != nil {
-		if errors.Is(err, context.Canceled) {
+		switch {
+		case errors.Is(err, context.Canceled):
 			return nil
+		case strings.Contains(err.Error(), "unsupported model format"):
+			// pull and retry to see if the model has been updated
+			parts := strings.Split(opts.Model, string(os.PathSeparator))
+			if len(parts) == 1 {
+				// this is a library model, log some info
+				fmt.Fprintln(os.Stderr, "This model is no longer compatible with Ollama. Pulling a new version...")
+			}
+			if err := PullHandler(cmd, []string{opts.Model}); err != nil {
+				fmt.Printf("Error: %s\n", err)
+				return fmt.Errorf("unsupported model, please update this model to gguf format") // relay the original error
+			}
+			// retry
+			if err := client.Generate(ctx, &request, fn); err != nil {
+				if errors.Is(err, context.Canceled) {
+					return nil
+				}
+				return err
+			}
+		default:
+			return err
 		}
-		return err
 	}
 	if opts.Prompt != "" {
 		fmt.Println()

docs/modelfile.md

@@ -188,7 +188,7 @@ SYSTEM """<system message>"""
 
 ### ADAPTER
 
-The `ADAPTER` instruction specifies the LoRA adapter to apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGML file format. The adapter should be tuned from the base model otherwise the behaviour is undefined.
+The `ADAPTER` instruction specifies the LoRA adapter to apply to the base model. The value of this instruction should be an absolute path or a path relative to the Modelfile and the file must be in a GGUF file format. The adapter should be tuned from the base model otherwise the behaviour is undefined.
 
 ```modelfile
 ADAPTER ./ollama-lora.bin

go.mod

@@ -12,6 +12,8 @@ require (
 
 require (
 	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 )
 
@@ -36,6 +38,7 @@ require (
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.4
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect

go.sum

@@ -100,6 +100,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=

gpu/gpu.go

@@ -0,0 +1,119 @@
+//go:build linux || windows
+
+package gpu
+
+/*
+#include "gpu_info.h"
+
+*/
+import "C"
+import (
+	"fmt"
+	"log"
+	"sync"
+	"unsafe"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+type handles struct {
+	cuda *C.cuda_handle_t
+	rocm *C.rocm_handle_t
+}
+
+var gpuMutex sync.Mutex
+var gpuHandles *handles = nil
+
+// Note: gpuMutex must already be held
+func initGPUHandles() {
+	log.Printf("Detecting GPU type")
+	gpuHandles = &handles{nil, nil}
+	var resp C.cuda_init_resp_t
+	C.cuda_init(&resp)
+	if resp.err != nil {
+		log.Printf("CUDA not detected: %s", C.GoString(resp.err))
+		C.free(unsafe.Pointer(resp.err))
+
+		var resp C.rocm_init_resp_t
+		C.rocm_init(&resp)
+		if resp.err != nil {
+			log.Printf("ROCm not detected: %s", C.GoString(resp.err))
+			C.free(unsafe.Pointer(resp.err))
+		} else {
+			log.Printf("Radeon GPU detected")
+			rocm := resp.rh
+			gpuHandles.rocm = &rocm
+		}
+	} else {
+		log.Printf("Nvidia GPU detected")
+		cuda := resp.ch
+		gpuHandles.cuda = &cuda
+	}
+}
+
+func GetGPUInfo() GpuInfo {
+	// TODO - consider exploring lspci (and equivalent on windows) to check for
+	// GPUs so we can report warnings if we see Nvidia/AMD but fail to load the libraries
+	gpuMutex.Lock()
+	defer gpuMutex.Unlock()
+	if gpuHandles == nil {
+		initGPUHandles()
+	}
+
+	var memInfo C.mem_info_t
+	var resp GpuInfo
+	if gpuHandles.cuda != nil {
+		C.cuda_check_vram(*gpuHandles.cuda, &memInfo)
+		resp.Driver = "CUDA"
+	} else if gpuHandles.rocm != nil {
+		C.rocm_check_vram(*gpuHandles.rocm, &memInfo)
+		resp.Driver = "ROCM"
+	} else {
+		C.cpu_check_ram(&memInfo)
+		resp.Driver = "CPU"
+	}
+	if memInfo.err != nil {
+		log.Printf("error looking up GPU memory: %s", C.GoString(memInfo.err))
+		C.free(unsafe.Pointer(memInfo.err))
+	}
+	resp.FreeMemory = uint64(memInfo.free)
+	resp.TotalMemory = uint64(memInfo.total)
+	return resp
+}
+
+func CheckVRAM() (int64, error) {
+	gpuInfo := GetGPUInfo()
+	if gpuInfo.FreeMemory > 0 && gpuInfo.Driver != "CPU" {
+		return int64(gpuInfo.FreeMemory), nil
+	}
+	return 0, fmt.Errorf("no GPU detected") // TODO - better handling of CPU based memory determiniation
+}
+
+func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
+	if opts.NumGPU != -1 {
+		return opts.NumGPU
+	}
+	info := GetGPUInfo()
+	if info.Driver == "CPU" {
+		return 0
+	}
+
+	/*
+		Calculate bytes per layer, this will roughly be the size of the model file divided by the number of layers.
+		We can store the model weights and the kv cache in vram,
+		to enable kv chache vram storage add two additional layers to the number of layers retrieved from the model file.
+	*/
+	bytesPerLayer := uint64(fileSizeBytes / numLayer)
+
+	// 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors
+	layers := int(info.FreeMemory/bytesPerLayer) * 3 / 4
+
+	// TODO - not sure on this part... if we can't fit all the layers, just fallback to CPU
+	// if int64(layers) < numLayer {
+	// 	log.Printf("%d MB VRAM available, insufficient to load current model (reuires %d MB) - falling back to CPU %d", freeBytes/(1024*1024), fileSizeBytes/(1024*1024))
+	// 	return 0
+	// }
+	log.Printf("%d MB VRAM available, loading up to %d GPU layers out of %d", info.FreeMemory/(1024*1024), layers, numLayer)
+
+	return layers
+}

gpu/gpu_darwin.go

@@ -0,0 +1,34 @@
+//go:build darwin
+
+package gpu
+
+import "C"
+import (
+	"github.com/jmorganca/ollama/api"
+)
+
+// CheckVRAM returns the free VRAM in bytes on Linux machines with NVIDIA GPUs
+func CheckVRAM() (int64, error) {
+	// TODO - assume metal, and return free memory?
+	return 0, nil
+
+}
+
+func GetGPUInfo() GpuInfo {
+	// TODO - Metal vs. x86 macs...
+
+	return GpuInfo{
+		Driver:      "METAL",
+		TotalMemory: 0,
+		FreeMemory:  0,
+	}
+}
+
+func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
+	// default to enable metal on macOS
+	return 1
+}
+
+func nativeInit() error {
+	return nil
+}

gpu/gpu_info.h

@@ -0,0 +1,49 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_H__
+#define __GPU_INFO_H__
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifndef _WIN32
+#include <dlfcn.h>
+#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
+#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
+#define LOAD_ERR() dlerror()
+#define UNLOAD_LIBRARY(handle) dlclose(handle)
+#else
+#include <windows.h>
+#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
+#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
+#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
+
+// TODO - refactor this with proper error message handling on windows
+inline static char *LOAD_ERR() {
+  static char errbuf[8];
+  snprintf(errbuf, 8, "0x%lx", GetLastError());
+  return errbuf;
+}
+
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct mem_info {
+  uint64_t total;
+  uint64_t free;
+  char *err;  // If non-nill, caller responsible for freeing
+} mem_info_t;
+
+void cpu_check_ram(mem_info_t *resp);
+
+#ifdef __cplusplus
+}
+#endif
+
+#include "gpu_info_cuda.h"
+#include "gpu_info_rocm.h"
+
+#endif  // __GPU_INFO_H__
+#endif  // __APPLE__

gpu/gpu_info_cpu.c

@@ -0,0 +1,42 @@
+#include "gpu_info.h"
+// Fallbacks for CPU mode
+
+#ifdef _WIN32
+#include <sysinfoapi.h>
+void cpu_check_ram(mem_info_t *resp) {
+  resp->err = NULL;
+  MEMORYSTATUSEX info;
+  if (GlobalMemoryStatusEx(&info) != 0) {
+    resp->total = info.ullTotalPhys;
+    resp->free = info.ullAvailPhys;
+  } else {
+    resp->err = strdup(LOAD_ERR());
+  }
+  return;
+}
+
+#elif __linux__
+#include <errno.h>
+#include <string.h>
+#include <sys/sysinfo.h>
+void cpu_check_ram(mem_info_t *resp) {
+  struct sysinfo info;
+  resp->err = NULL;
+  if (sysinfo(&info) != 0) {
+    resp->err = strdup(strerror(errno));
+  } else {
+    resp->total = info.totalram * info.mem_unit;
+    resp->free = info.freeram * info.mem_unit;
+  }
+  return;
+}
+
+#elif __APPLE__
+// TODO consider an Apple implementation that does something useful
+// mem_info_t cpu_check_ram() {
+//   mem_info_t resp = {0, 0, NULL};
+//   return resp;
+// }
+#else
+#error "Unsupported platform"
+#endif

gpu/gpu_info_cuda.c

@@ -0,0 +1,111 @@
+#ifndef __APPLE__  // TODO - maybe consider nvidia support on intel macs?
+
+#include "gpu_info_cuda.h"
+
+#include <string.h>
+
+#ifndef _WIN32
+const char *cuda_lib_paths[] = {
+    "libnvidia-ml.so",
+    "/usr/local/cuda/lib64/libnvidia-ml.so",
+    NULL,
+};
+#else
+const char *cuda_lib_paths[] = {
+    "nvml.dll",
+    "",
+    NULL,
+};
+#endif
+
+void cuda_init(cuda_init_resp_t *resp) {
+  resp->err = NULL;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i;
+
+  struct lookup {
+    char *s;
+    void **p;
+  } l[4] = {
+      {"nvmlInit_v2", (void *)&resp->ch.initFn},
+      {"nvmlShutdown", (void *)&resp->ch.shutdownFn},
+      {"nvmlDeviceGetHandleByIndex", (void *)&resp->ch.getHandle},
+      {"nvmlDeviceGetMemoryInfo", (void *)&resp->ch.getMemInfo},
+  };
+
+  for (i = 0; cuda_lib_paths[i] != NULL && resp->ch.handle == NULL; i++) {
+    resp->ch.handle = LOAD_LIBRARY(cuda_lib_paths[i], RTLD_LAZY);
+  }
+  if (!resp->ch.handle) {
+    snprintf(buf, buflen,
+             "Unable to load %s library to query for Nvidia GPUs: %s",
+             cuda_lib_paths[0], LOAD_ERR());
+    resp->err = strdup(buf);
+    return;
+  }
+
+  for (i = 0; i < 4; i++) {  // TODO - fix this to use a null terminated list
+    *l[i].p = LOAD_SYMBOL(resp->ch.handle, l[i].s);
+    if (!l[i].p) {
+      UNLOAD_LIBRARY(resp->ch.handle);
+      resp->ch.handle = NULL;
+      snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
+               LOAD_ERR());
+      resp->err = strdup(buf);
+      return;
+    }
+  }
+  return;
+}
+
+void cuda_check_vram(cuda_handle_t h, mem_info_t *resp) {
+  resp->err = NULL;
+  nvmlDevice_t device;
+  nvmlMemory_t memInfo = {0};
+  nvmlReturn_t ret;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i;
+
+  if (h.handle == NULL) {
+    resp->err = strdup("nvml handle sn't initialized");
+    return;
+  }
+
+  // printf("Initializing nvidia-ml library\n");
+  ret = (*h.initFn)();
+  if (ret != NVML_SUCCESS) {
+    snprintf(buf, buflen, "nvml vram init failure: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+
+  // TODO - handle multiple GPUs
+  ret = (*h.getHandle)(0, &device);
+  if (ret != NVML_SUCCESS) {
+    (*h.shutdownFn)();
+    snprintf(buf, buflen, "unable to get device handle: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+
+  ret = (*h.getMemInfo)(device, &memInfo);
+  if (ret != NVML_SUCCESS) {
+    (*h.shutdownFn)();
+    snprintf(buf, buflen, "device memory info lookup failure: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+  resp->total = memInfo.total;
+  resp->free = memInfo.free;
+
+  ret = (*h.shutdownFn)();
+  if (ret != NVML_SUCCESS) {
+    snprintf(buf, buflen, "nvml vram shutdown failure: %d", ret);
+    resp->err = strdup(buf);
+  }
+
+  return;
+}
+#endif  // __APPLE__

gpu/gpu_info_cuda.h

@@ -0,0 +1,35 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_CUDA_H__
+#define __GPU_INFO_CUDA_H__
+#include "gpu_info.h"
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum nvmlReturn_enum {
+  NVML_SUCCESS = 0,
+  // Other values omitted for now...
+} nvmlReturn_t;
+typedef void *nvmlDevice_t;  // Opaque is sufficient
+typedef struct nvmlMemory_st {
+  unsigned long long total;
+  unsigned long long free;
+  unsigned long long used;
+} nvmlMemory_t;
+
+typedef struct cuda_handle {
+  void *handle;
+  nvmlReturn_t (*initFn)(void);
+  nvmlReturn_t (*shutdownFn)(void);
+  nvmlReturn_t (*getHandle)(unsigned int, nvmlDevice_t *);
+  nvmlReturn_t (*getMemInfo)(nvmlDevice_t, nvmlMemory_t *);
+} cuda_handle_t;
+
+typedef struct cuda_init_resp {
+  char *err;  // If err is non-null handle is invalid
+  cuda_handle_t ch;
+} cuda_init_resp_t;
+
+void cuda_init(cuda_init_resp_t *resp);
+void cuda_check_vram(cuda_handle_t ch, mem_info_t *resp);
+
+#endif  // __GPU_INFO_CUDA_H__
+#endif  // __APPLE__

gpu/gpu_info_rocm.c

@@ -0,0 +1,112 @@
+#ifndef __APPLE__
+
+#include "gpu_info_rocm.h"
+
+#include <string.h>
+
+#ifndef _WIN32
+const char *rocm_lib_paths[] = {
+    "librocm_smi64.so",
+    "/opt/rocm/lib/librocm_smi64.so",
+    NULL,
+};
+#else
+// TODO untested
+const char *rocm_lib_paths[] = {
+    "rocm_smi64.dll",
+    "/opt/rocm/lib/rocm_smi64.dll",
+    NULL,
+};
+#endif
+
+void rocm_init(rocm_init_resp_t *resp) {
+  resp->err = NULL;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i;
+  struct lookup {
+    char *s;
+    void **p;
+  } l[4] = {
+      {"rsmi_init", (void *)&resp->rh.initFn},
+      {"rsmi_shut_down", (void *)&resp->rh.shutdownFn},
+      {"rsmi_dev_memory_total_get", (void *)&resp->rh.totalMemFn},
+      {"rsmi_dev_memory_usage_get", (void *)&resp->rh.usageMemFn},
+      // { "rsmi_dev_id_get", (void*)&resp->rh.getHandle },
+  };
+
+  for (i = 0; rocm_lib_paths[i] != NULL && resp->rh.handle == NULL; i++) {
+    resp->rh.handle = LOAD_LIBRARY(rocm_lib_paths[i], RTLD_LAZY);
+  }
+  if (!resp->rh.handle) {
+    snprintf(buf, buflen,
+             "Unable to load %s library to query for Radeon GPUs: %s\n",
+             rocm_lib_paths[0], LOAD_ERR());
+    resp->err = strdup(buf);
+    return;
+  }
+
+  for (i = 0; i < 4; i++) {
+    *l[i].p = LOAD_SYMBOL(resp->rh.handle, l[i].s);
+    if (!l[i].p) {
+      UNLOAD_LIBRARY(resp->rh.handle);
+      snprintf(buf, buflen, "symbol lookup for %s failed: %s", l[i].s,
+               LOAD_ERR());
+      resp->err = strdup(buf);
+      return;
+    }
+  }
+  return;
+}
+
+void rocm_check_vram(rocm_handle_t h, mem_info_t *resp) {
+  resp->err = NULL;
+  // uint32_t num_devices;
+  // uint16_t device;
+  uint64_t totalMem = 0;
+  uint64_t usedMem = 0;
+  rsmi_status_t ret;
+  const int buflen = 256;
+  char buf[buflen + 1];
+  int i;
+
+  // printf("Initializing rocm smi library\n");
+  ret = (*h.initFn)(0);
+  if (ret != RSMI_STATUS_SUCCESS) {
+    snprintf(buf, buflen, "rocm vram init failure: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+
+  // TODO - iterate through devices...  ret =
+  // rsmi_num_monitor_devices(&num_devices);
+
+  // ret = (*h.getHandle)(0, &device);
+  // if (ret != RSMI_STATUS_SUCCESS) {
+  //     printf("rocm vram device lookup failure: %d\n", ret);
+  //     return -1;
+  // }
+
+  // Get total memory - used memory for available memory
+  ret = (*h.totalMemFn)(0, RSMI_MEM_TYPE_VRAM, &totalMem);
+  if (ret != RSMI_STATUS_SUCCESS) {
+    (*h.shutdownFn)();
+    snprintf(buf, buflen, "rocm total mem lookup failure: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+  ret = (*h.usageMemFn)(0, RSMI_MEM_TYPE_VRAM, &usedMem);
+  if (ret != RSMI_STATUS_SUCCESS) {
+    (*h.shutdownFn)();
+    snprintf(buf, buflen, "rocm usage mem lookup failure: %d", ret);
+    resp->err = strdup(buf);
+    return;
+  }
+
+  (*h.shutdownFn)();
+  resp->total = totalMem;
+  resp->free = totalMem - usedMem;
+  return;
+}
+
+#endif  // __APPLE__

gpu/gpu_info_rocm.h

@@ -0,0 +1,36 @@
+#ifndef __APPLE__
+#ifndef __GPU_INFO_ROCM_H__
+#define __GPU_INFO_ROCM_H__
+#include "gpu_info.h"
+
+// Just enough typedef's to dlopen/dlsym for memory information
+typedef enum rsmi_status_return {
+  RSMI_STATUS_SUCCESS = 0,
+  // Other values omitted for now...
+} rsmi_status_t;
+
+typedef enum rsmi_memory_type {
+  RSMI_MEM_TYPE_VRAM = 0,
+  RSMI_MEM_TYPE_VIS_VRAM,
+  RSMI_MEM_TYPE_GTT,
+} rsmi_memory_type_t;
+
+typedef struct rocm_handle {
+  void *handle;
+  rsmi_status_t (*initFn)(uint64_t);
+  rsmi_status_t (*shutdownFn)(void);
+  rsmi_status_t (*totalMemFn)(uint32_t, rsmi_memory_type_t, uint64_t *);
+  rsmi_status_t (*usageMemFn)(uint32_t, rsmi_memory_type_t, uint64_t *);
+  // rsmi_status_t (*getHandle)(uint32_t, uint16_t *);
+} rocm_handle_t;
+
+typedef struct rocm_init_resp {
+  char *err;  // If err is non-null handle is invalid
+  rocm_handle_t rh;
+} rocm_init_resp_t;
+
+void rocm_init(rocm_init_resp_t *resp);
+void rocm_check_vram(rocm_handle_t rh, mem_info_t *resp);
+
+#endif  // __GPU_INFO_ROCM_H__
+#endif  // __APPLE__

gpu/gpu_test.go

@@ -0,0 +1,26 @@
+package gpu
+
+import (
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBasicGetGPUInfo(t *testing.T) {
+	info := GetGPUInfo()
+	assert.Contains(t, "CUDA ROCM CPU METAL", info.Driver)
+
+	switch runtime.GOOS {
+	case "darwin":
+		// TODO - remove this once MacOS returns some size for CPU
+		return
+	case "linux", "windows":
+		assert.Greater(t, info.TotalMemory, uint64(0))
+		assert.Greater(t, info.FreeMemory, uint64(0))
+	default:
+		return
+	}
+}
+
+// TODO - add some logic to figure out card type through other means and actually verify we got back what we expected

gpu/types.go

@@ -0,0 +1,10 @@
+package gpu
+
+// Beginning of an `ollama info` command
+type GpuInfo struct {
+	Driver      string `json:"driver,omitempty"`
+	TotalMemory uint64 `json:"total_memory,omitempty"`
+	FreeMemory  uint64 `json:"free_memory,omitempty"`
+
+	// TODO add other useful attributes about the card here for discovery information
+}

llm/ext_server.go

@@ -0,0 +1,439 @@
+package llm
+
+/*
+#cgo CFLAGS: -I${SRCDIR}/llama.cpp/gguf -I${SRCDIR}/llama.cpp/gguf/common -I${SRCDIR}/llama.cpp/gguf/examples/server
+#cgo CFLAGS: -DNDEBUG -DLLAMA_SERVER_LIBRARY=1 -D_XOPEN_SOURCE=600 -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
+#cgo CFLAGS: -Wmissing-noreturn -Wall -Wextra -Wcast-qual -Wno-unused-function -Wno-array-bounds
+#cgo CPPFLAGS: -Ofast -Wall -Wextra -Wno-unused-function -Wno-unused-variable -Wno-deprecated-declarations -Wno-unused-but-set-variable
+#cgo darwin CFLAGS: -D_DARWIN_C_SOURCE
+#cgo darwin CPPFLAGS:  -DGGML_USE_ACCELERATE
+#cgo darwin,arm64 CPPFLAGS: -DGGML_USE_METAL -DGGML_METAL_NDEBUG
+#cgo darwin LDFLAGS: -lc++ -framework Accelerate
+#cgo darwin,arm64 LDFLAGS: -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
+#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/common/libcommon.a
+#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/examples/server/libext_server.a
+#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libllama.a
+#cgo darwin,arm64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/metal/libggml_static.a
+#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/common/libcommon.a
+#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/examples/server/libext_server.a
+#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libllama.a
+#cgo darwin,amd64 LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cpu/libggml_static.a
+#cgo linux CFLAGS: -D_GNU_SOURCE
+#cgo linux windows CFLAGS: -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_MMV_Y=1 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_USE_CUBLAS
+#cgo linux LDFLAGS: -L/usr/local/cuda/targets/x86_64-linux/lib -L/usr/local/cuda/lib64 -L/usr/local/cuda/targets/x86_64-linux/lib/stubs
+
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cuda/examples/server/libext_server.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cuda/common/libcommon.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cuda/libllama.a
+#cgo linux LDFLAGS: ${SRCDIR}/llama.cpp/gguf/build/cuda/libggml_static.a
+
+// Note: the following requires cuda library presence on linux to build, even if you only have rocm
+#cgo linux LDFLAGS: /usr/local/cuda/lib64/libcudart_static.a
+#cgo linux LDFLAGS: /usr/local/cuda/lib64/libcublas_static.a
+#cgo linux LDFLAGS: /usr/local/cuda/lib64/libcublasLt_static.a
+#cgo linux LDFLAGS: /usr/local/cuda/lib64/libcudadevrt.a
+#cgo linux LDFLAGS: /usr/local/cuda/lib64/libculibos.a
+#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm
+#cgo windows LDFLAGS: -L${SRCDIR}/llama.cpp/gguf/build/wincuda/dist/bin
+#cgo windows LDFLAGS: -lext_server_shared -lpthread
+
+#include <stdlib.h>
+#include "server.h"
+
+*/
+import "C"
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"time"
+	"unsafe"
+
+	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/gpu"
+)
+
+func newExtServerResp(len C.size_t) C.ext_server_resp_t {
+	var resp C.ext_server_resp_t
+	resp.msg_len = len
+	bytes := make([]byte, len)
+	resp.msg = (*C.char)(C.CBytes(bytes))
+	return resp
+}
+
+func freeExtServerResp(resp C.ext_server_resp_t) {
+	if resp.msg_len == 0 {
+		return
+	}
+	C.free(unsafe.Pointer(resp.msg))
+}
+
+func extServerResponseToErr(resp C.ext_server_resp_t) error {
+	return fmt.Errorf(C.GoString(resp.msg))
+}
+
+type extServer interface {
+	LLM
+	llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t)
+	llama_server_start()
+	llama_server_stop()
+	llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t)
+	llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t)
+	llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t)
+	llama_server_release_task_result(result *C.ext_server_task_result_t)
+	llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t)
+	llama_server_release_json_resp(json_resp **C.char)
+}
+
+type llamaExtServer struct {
+	api.Options
+}
+
+// Note: current implementation does not support concurrent instantiations
+var mutex sync.Mutex
+
+func (llm *llamaExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
+	C.llama_server_init(sparams, err)
+}
+func (llm *llamaExtServer) llama_server_start() {
+	C.llama_server_start()
+}
+func (llm *llamaExtServer) llama_server_stop() {
+	C.llama_server_stop()
+}
+
+func (llm *llamaExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
+	C.llama_server_completion(json_req, resp)
+}
+func (llm *llamaExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
+	C.llama_server_completion_next_result(task_id, resp)
+}
+func (llm *llamaExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
+	C.llama_server_completion_cancel(task_id, err)
+}
+func (llm *llamaExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
+	C.llama_server_release_task_result(result)
+}
+
+func (llm *llamaExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_tokenize(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_detokenize(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.llama_server_embedding(json_req, json_resp, err)
+}
+func (llm *llamaExtServer) llama_server_release_json_resp(json_resp **C.char) {
+	C.llama_server_release_json_resp(json_resp)
+}
+
+func newLlamaExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	server := &llamaExtServer{opts}
+	return newExtServer(server, model, adapters, projectors, numLayers, opts)
+}
+
+func newExtServer(server extServer, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	if !mutex.TryLock() {
+		log.Printf("concurrent llm servers not yet supported, waiting for prior server to complete")
+		mutex.Lock()
+	}
+	fileInfo, err := os.Stat(model)
+	if err != nil {
+		return nil, err
+	}
+	var sparams C.ext_server_params_t
+	sparams.model = C.CString(model)
+	defer C.free(unsafe.Pointer(sparams.model))
+
+	numGPU := gpu.NumGPU(numLayers, fileInfo.Size(), opts)
+
+	sparams.embedding = true
+	sparams.n_ctx = C.uint(opts.NumCtx)
+	sparams.n_batch = C.uint(opts.NumBatch)
+	sparams.n_gpu_layers = C.int(numGPU)
+	sparams.main_gpu = C.int(opts.MainGPU)
+	sparams.n_parallel = 2 // TODO - wire up concurrency
+
+	// Always use the value encoded in the model
+	sparams.rope_freq_base = 0.0
+	sparams.rope_freq_scale = 0.0
+	sparams.memory_f16 = C.bool(opts.F16KV)
+	sparams.use_mlock = C.bool(opts.UseMLock)
+	sparams.use_mmap = C.bool(opts.UseMMap)
+	sparams.numa = C.bool(opts.UseNUMA)
+
+	sparams.lora_adapters = nil
+	for i := 0; i < len(adapters); i++ {
+		la := (*C.ext_server_lora_adapter_t)(C.malloc(C.sizeof_ext_server_lora_adapter_t))
+		defer C.free(unsafe.Pointer(la))
+		la.adapter = C.CString(adapters[i])
+		defer C.free(unsafe.Pointer(la.adapter))
+		la.scale = C.float(1.0) // TODO expose scale/weights up through ollama UX
+		la.next = nil
+		if i == 0 {
+			sparams.lora_adapters = la
+		} else {
+			tmp := sparams.lora_adapters
+			for ; tmp.next != nil; tmp = tmp.next {
+			}
+			tmp.next = la
+		}
+	}
+
+	if len(projectors) > 0 {
+		// TODO: applying multiple projectors is not supported by the llama.cpp server yet
+		sparams.mmproj = C.CString(projectors[0])
+		defer C.free(unsafe.Pointer(sparams.mmproj))
+	} else {
+		sparams.mmproj = nil
+	}
+
+	if opts.NumThread > 0 {
+		sparams.n_threads = C.uint(opts.NumThread)
+	} else {
+		sparams.n_threads = C.uint(runtime.NumCPU())
+	}
+
+	log.Printf("Initializing internal llama server")
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	server.llama_server_init(&sparams, &resp)
+	if resp.id < 0 {
+		return nil, extServerResponseToErr(resp)
+	}
+
+	log.Printf("Starting internal llama main loop")
+	server.llama_server_start()
+	return server, nil
+}
+
+func (llm *llamaExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
+	return predict(llm, llm.Options, ctx, pred, fn)
+}
+
+func predict(llm extServer, opts api.Options, ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	request := map[string]any{
+		"prompt":            predict.Prompt,
+		"stream":            true,
+		"n_predict":         opts.NumPredict,
+		"n_keep":            opts.NumKeep,
+		"temperature":       opts.Temperature,
+		"top_k":             opts.TopK,
+		"top_p":             opts.TopP,
+		"tfs_z":             opts.TFSZ,
+		"typical_p":         opts.TypicalP,
+		"repeat_last_n":     opts.RepeatLastN,
+		"repeat_penalty":    opts.RepeatPenalty,
+		"presence_penalty":  opts.PresencePenalty,
+		"frequency_penalty": opts.FrequencyPenalty,
+		"mirostat":          opts.Mirostat,
+		"mirostat_tau":      opts.MirostatTau,
+		"mirostat_eta":      opts.MirostatEta,
+		"penalize_nl":       opts.PenalizeNewline,
+		"seed":              opts.Seed,
+		"stop":              opts.Stop,
+	}
+
+	if predict.Format == "json" {
+		request["grammar"] = jsonGrammar
+	}
+
+	retryDelay := 100 * time.Microsecond
+	for retries := 0; retries < maxRetries; retries++ {
+		if retries > 0 {
+			time.Sleep(retryDelay) // wait before retrying
+			retryDelay *= 2        // exponential backoff
+		}
+
+		// Handling JSON marshaling with special characters unescaped.
+		buffer := &bytes.Buffer{}
+		enc := json.NewEncoder(buffer)
+		enc.SetEscapeHTML(false)
+
+		if err := enc.Encode(request); err != nil {
+			return fmt.Errorf("failed to marshal data: %w", err)
+		}
+
+		req := C.CString(buffer.String())
+		defer C.free(unsafe.Pointer(req))
+
+		llm.llama_server_completion(req, &resp)
+		if resp.id < 0 {
+			return extServerResponseToErr(resp)
+		}
+
+		retryNeeded := false
+	out:
+		for {
+			select {
+			case <-ctx.Done():
+				// This handles the request cancellation
+				llm.llama_server_completion_cancel(resp.id, &resp)
+				if resp.id < 0 {
+					return extServerResponseToErr(resp)
+				} else {
+					return nil
+				}
+			default:
+				var result C.ext_server_task_result_t
+				llm.llama_server_completion_next_result(resp.id, &result)
+				json_resp := C.GoString(result.json_resp)
+				llm.llama_server_release_task_result(&result)
+
+				var p prediction
+				if err := json.Unmarshal([]byte(json_resp), &p); err != nil {
+					llm.llama_server_completion_cancel(resp.id, &resp)
+					if resp.id < 0 {
+						return fmt.Errorf("error unmarshaling llm prediction response: %w and cancel %s", err, C.GoString(resp.msg))
+					} else {
+						return fmt.Errorf("error unmarshaling llm prediction response: %w", err)
+					}
+				}
+
+				if bool(result.error) && strings.Contains(json_resp, "slot unavailable") {
+					retryNeeded = true
+					// task will already be canceled
+					break out
+				}
+
+				if p.Content != "" {
+					fn(PredictResult{
+						CreatedAt: time.Now().UTC(),
+						Content:   p.Content,
+					})
+				}
+
+				if p.Stop {
+					fn(PredictResult{
+						CreatedAt:          time.Now().UTC(),
+						TotalDuration:      time.Since(predict.CheckpointStart),
+						Done:               true,
+						PromptEvalCount:    p.Timings.PromptN,
+						PromptEvalDuration: parseDurationMs(p.Timings.PromptMS),
+						EvalCount:          p.Timings.PredictedN,
+						EvalDuration:       parseDurationMs(p.Timings.PredictedMS),
+					})
+					return nil
+				}
+			}
+		}
+		if !retryNeeded {
+			return nil // success
+		}
+	}
+
+	// should never reach here ideally
+	return fmt.Errorf("max retries exceeded")
+}
+func (llm *llamaExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
+	return encode(llm, ctx, prompt)
+}
+
+func encode(llm extServer, ctx context.Context, prompt string) ([]int, error) {
+	data, err := json.Marshal(TokenizeRequest{Content: prompt})
+	if err != nil {
+		return nil, fmt.Errorf("marshaling encode data: %w", err)
+	}
+	req := C.CString(string(data))
+	defer C.free(unsafe.Pointer(req))
+	var json_resp *C.char
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	llm.llama_server_tokenize(req, &json_resp, &resp)
+	if resp.id < 0 {
+		return nil, extServerResponseToErr(resp)
+	}
+	defer llm.llama_server_release_json_resp(&json_resp)
+
+	var encoded TokenizeResponse
+	if err2 := json.Unmarshal([]byte(C.GoString(json_resp)), &encoded); err2 != nil {
+		return nil, fmt.Errorf("unmarshal encode response: %w", err2)
+	}
+
+	return encoded.Tokens, err
+}
+
+func (llm *llamaExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
+	return decode(llm, ctx, tokens)
+}
+
+func decode(llm extServer, ctx context.Context, tokens []int) (string, error) {
+	if len(tokens) == 0 {
+		return "", nil
+	}
+	data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
+	if err != nil {
+		return "", fmt.Errorf("marshaling decode data: %w", err)
+	}
+
+	req := C.CString(string(data))
+	defer C.free(unsafe.Pointer(req))
+	var json_resp *C.char
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	llm.llama_server_detokenize(req, &json_resp, &resp)
+	if resp.id < 0 {
+		return "", extServerResponseToErr(resp)
+	}
+	defer llm.llama_server_release_json_resp(&json_resp)
+
+	var decoded DetokenizeResponse
+	if err2 := json.Unmarshal([]byte(C.GoString(json_resp)), &decoded); err2 != nil {
+		return "", fmt.Errorf("unmarshal encode response: %w", err2)
+	}
+
+	return decoded.Content, err
+}
+
+func (llm *llamaExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
+	return embedding(llm, ctx, input)
+}
+func embedding(llm extServer, ctx context.Context, input string) ([]float64, error) {
+	data, err := json.Marshal(TokenizeRequest{Content: input})
+	if err != nil {
+		return nil, fmt.Errorf("error marshaling embed data: %w", err)
+	}
+
+	req := C.CString(string(data))
+	defer C.free(unsafe.Pointer(req))
+	var json_resp *C.char
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	llm.llama_server_embedding(req, &json_resp, &resp)
+	if resp.id < 0 {
+		return nil, extServerResponseToErr(resp)
+	}
+	defer llm.llama_server_release_json_resp(&json_resp)
+
+	var embedding EmbeddingResponse
+	if err := json.Unmarshal([]byte(C.GoString(json_resp)), &embedding); err != nil {
+		return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
+	}
+
+	return embedding.Embedding, nil
+}
+
+func (llm *llamaExtServer) Ping(ctx context.Context) error {
+	// TODO - consider some mechanism to check if the main loop and llama.cpp are in a good state
+	return nil
+}
+
+func (llm *llamaExtServer) Close() {
+	close(llm)
+}
+
+func close(llm extServer) {
+	llm.llama_server_stop()
+	mutex.Unlock()
+}

llm/ggml.go

@@ -86,74 +86,6 @@ type container interface {
 	Decode(*readSeekOffset) (model, error)
 }
 
-type containerGGML struct{}
-
-func (c *containerGGML) Name() string {
-	return "ggml"
-}
-
-func (c *containerGGML) Decode(ro *readSeekOffset) (model, error) {
-	// file contents aren't decoded
-	ro.Seek(0, io.SeekEnd)
-	return nil, nil
-}
-
-type containerGGMF struct {
-	version uint32
-}
-
-func (c *containerGGMF) Name() string {
-	return "ggmf"
-}
-
-func (c *containerGGMF) Decode(ro *readSeekOffset) (model, error) {
-	var version uint32
-	binary.Read(ro, binary.LittleEndian, &version)
-
-	switch version {
-	case 1:
-	default:
-		return nil, errors.New("invalid version")
-	}
-
-	c.version = version
-
-	// remaining file contents aren't decoded
-	ro.Seek(0, io.SeekEnd)
-
-	return nil, nil
-}
-
-type containerGGJT struct {
-	version uint32
-}
-
-func (c *containerGGJT) Name() string {
-	return "ggjt"
-}
-
-func (c *containerGGJT) Decode(ro *readSeekOffset) (model, error) {
-	var version uint32
-	binary.Read(ro, binary.LittleEndian, &version)
-
-	switch version {
-	case 1, 2, 3:
-	default:
-		return nil, errors.New("invalid version")
-	}
-
-	c.version = version
-
-	// different model types may have different layouts for hyperparameters
-	var llama llamaModel
-	binary.Read(ro, binary.LittleEndian, &llama.hyperparameters)
-
-	// remaining file contents aren't decoded
-	ro.Seek(0, io.SeekEnd)
-
-	return &llama, nil
-}
-
 type containerLORA struct {
 	version uint32
 }
@@ -194,6 +126,8 @@ const (
 	FILE_MAGIC_GGUF_BE = 0x47475546
 )
 
+var ErrUnsupportedFormat = errors.New("unsupported model format")
+
 func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
 	ro := readSeekOffset{ReadSeeker: r}
 
@@ -204,12 +138,8 @@ func DecodeGGML(r io.ReadSeeker) (*GGML, error) {
 
 	var c container
 	switch magic {
-	case FILE_MAGIC_GGML:
-		c = &containerGGML{}
-	case FILE_MAGIC_GGMF:
-		c = &containerGGMF{}
-	case FILE_MAGIC_GGJT:
-		c = &containerGGJT{}
+	case FILE_MAGIC_GGML, FILE_MAGIC_GGMF, FILE_MAGIC_GGJT:
+		return nil, ErrUnsupportedFormat
 	case FILE_MAGIC_GGLA:
 		c = &containerLORA{}
 	case FILE_MAGIC_GGUF_LE:

llm/llama.cpp/gen_common.sh

@@ -0,0 +1,35 @@
+# common logic accross linux and darwin
+
+init_vars() {
+    LLAMACPP_DIR=gguf
+    PATCHES="0001-Expose-callable-API-for-server.patch"
+    CMAKE_DEFS="-DLLAMA_ACCELERATE=on"
+    # TODO - LLAMA_K_QUANTS is stale and needs to be mapped to newer cmake settings
+    CMAKE_TARGETS="--target ggml --target ggml_static --target llama --target build_info --target common --target ext_server --target llava_static"
+    if echo "${CGO_CFLAGS}" | grep -- '-g' > /dev/null ; then
+        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_GPROF=on ${CMAKE_DEFS}"
+    else
+        # TODO - add additional optimization flags...
+        CMAKE_DEFS="-DCMAKE_BUILD_TYPE=Release ${CMAKE_DEFS}"
+    fi
+}
+
+git_module_setup() {
+    # TODO add flags to skip the init/patch logic to make it easier to mod llama.cpp code in-repo
+    git submodule init
+    git submodule update --force gguf
+
+}
+
+apply_patches() {
+    # Workaround git apply not handling creation well for iteration
+    rm -f gguf/examples/server/server.h
+    for patch in ${PATCHES} ; do
+        git -C gguf apply ../patches/${patch}
+    done
+}
+
+build() {
+    cmake -S ${LLAMACPP_DIR} -B ${BUILD_DIR} ${CMAKE_DEFS}
+    cmake --build ${BUILD_DIR} ${CMAKE_TARGETS} -j8
+}

llm/llama.cpp/gen_darwin.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# This script is intended to run inside the go generate
+# working directory must be ../llm/llama.cpp
+
+# TODO - add hardening to detect missing tools (cmake, etc.)
+
+set -ex
+set -o pipefail
+echo "Starting darwin generate script"
+source $(dirname $0)/gen_common.sh
+init_vars
+CMAKE_DEFS="-DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 ${CMAKE_DEFS}"
+case "${GOARCH}" in
+    "amd64")
+        CMAKE_DEFS="-DLLAMA_METAL=off -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 ${CMAKE_DEFS}"
+        BUILD_DIR="gguf/build/cpu"
+        ;;
+     "arm64")
+        CMAKE_DEFS="-DLLAMA_METAL=on -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 ${CMAKE_DEFS}"
+        BUILD_DIR="gguf/build/metal"
+        ;;
+    *)
+        echo "GOARCH must be set"
+        echo "this script is meant to be run from within go generate"
+        exit 1
+        ;;
+esac
+
+git_module_setup
+apply_patches
+build
+
+# Enable local debug/run usecase
+if [ -e "gguf/ggml-metal.metal" ]; then
+    cp gguf/ggml-metal.metal ../../
+fi

llm/llama.cpp/gen_linux.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# This script is intended to run inside the go generate
+# working directory must be ../llm/llama.cpp
+
+set -ex
+set -o pipefail
+
+echo "Starting linux generate script"
+if [ -z "${CUDACXX}" -a -x /usr/local/cuda/bin/nvcc ] ; then
+    export CUDACXX=/usr/local/cuda/bin/nvcc
+fi
+source $(dirname $0)/gen_common.sh
+init_vars
+git_module_setup
+apply_patches
+CMAKE_DEFS="-DLLAMA_CUBLAS=on -DCMAKE_POSITION_INDEPENDENT_CODE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+BUILD_DIR="gguf/build/cuda"
+LIB_DIR="${BUILD_DIR}/lib"
+mkdir -p ../../dist/
+build
+# TODO - explore mechanism to soften the hard cuda dependency on linux
+#        by conditionally building some archive here that aggregates the cuda libs if present
+#        so that the cgo flags link this intermediate archive instead of the underlying cuda libs
+# 
+# gcc -fPIC -g -shared -o ${LIB_DIR}/libcuda_server.so \
+#     -Wl,--whole-archive \
+#     ${BUILD_DIR}/examples/server/CMakeFiles/ext_server.dir/server.cpp.o \
+#     ${BUILD_DIR}/common/libcommon.a \
+#     ${BUILD_DIR}/libllama.a \
+#     ${BUILD_DIR}/examples/llava/libllava_static.a \
+#     -Wl,--no-whole-archive \
+#     -lrt -lpthread -ldl -lstdc++ -lm \
+#     /usr/local/cuda/lib64/libcudart_static.a \
+#     /usr/local/cuda/lib64/libcublas_static.a \
+#     /usr/local/cuda/lib64/libcublasLt_static.a \
+#     /usr/local/cuda/lib64/libcudadevrt.a \
+#     /usr/local/cuda/lib64/libculibos.a
+
+if [ -z "${ROCM_PATH}" ] ; then
+    # Try the default location in case it exists
+    ROCM_PATH=/opt/rocm
+fi
+
+BUILD_DIR="gguf/build/rocm"
+LIB_DIR="${BUILD_DIR}/lib"
+mkdir -p ${LIB_DIR}
+# Ensure we have at least one file present for the embed
+touch ${LIB_DIR}/.generated 
+
+if [ -d "${ROCM_PATH}" ] ; then
+    echo "Building ROCm"
+    init_vars
+    CMAKE_DEFS="-DCMAKE_POSITION_INDEPENDENT_CODE=on -DCMAKE_VERBOSE_MAKEFILE=on -DLLAMA_HIPBLAS=on -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102' -DGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102'"
+    CMAKE_DEFS="-DLLAMA_ACCELERATE=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off ${CMAKE_DEFS}"
+    build
+    gcc -fPIC -g -shared -o ${LIB_DIR}/librocm_server.so \
+        -Wl,--whole-archive \
+        ${BUILD_DIR}/examples/server/libext_server.a \
+        ${BUILD_DIR}/common/libcommon.a \
+        ${BUILD_DIR}/libllama.a \
+        -Wl,--no-whole-archive \
+        -lrt -lpthread -ldl -lstdc++ -lm \
+        -L/opt/rocm/lib -L/opt/amdgpu/lib/x86_64-linux-gnu/ \
+        -Wl,-rpath,/opt/rocm/lib,-rpath,/opt/amdgpu/lib/x86_64-linux-gnu/ \
+        -lhipblas -lrocblas -lamdhip64 -lrocsolver -lamd_comgr -lhsa-runtime64 -lrocsparse -ldrm -ldrm_amdgpu
+fi

llm/llama.cpp/gen_windows.ps1

@@ -0,0 +1,55 @@
+#!powershell
+
+$ErrorActionPreference = "Stop"
+
+function init_vars {
+    $script:buildDir="gguf/build/wincuda"
+    $script:installDir="gguf/build/wincuda/dist"
+    $script:patches = @("0001-Expose-callable-API-for-server.patch")
+    $script:cmakeDefs = @("-DLLAMA_NATIVE=off", "-DLLAMA_F16C=off", "-DLLAMA_FMA=off", "-DLLAMA_AVX512=off", "-DLLAMA_AVX2=off", "-DLLAMA_AVX=on", "-DLLAMA_K_QUANTS=on", "-DLLAMA_ACCELERATE=on", "-DLLAMA_CUBLAS=ON","-DCMAKE_VERBOSE_MAKEFILE=ON","-DBUILD_SHARED_LIBS=on","-A","x64")
+
+    if ($env:CGO_CFLAGS -contains "-g") {
+        $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on")
+        $script:config += "RelWithDebInfo"
+    } else {
+        $script:config += "Release"
+    }
+}
+
+function git_module_setup {
+    # TODO add flags to skip the init/patch logic to make it easier to mod llama.cpp code in-repo
+    & git submodule init
+    & git submodule update --force gguf
+}
+
+function apply_patches {
+    rm -erroraction ignore -path "gguf/examples/server/server.h"
+    foreach ($patch in $patches) {
+        write-host "Applying patch $patch"
+        & git -C gguf apply ../patches/$patch
+    }
+}
+
+function build {
+    write-host "generating config with: cmake -S gguf -B $buildDir $cmakeDefs"
+    & cmake --version
+    & cmake -S gguf -B $buildDir $cmakeDefs
+    write-host "building with: cmake --build $buildDir --config $config"
+    & cmake --build $buildDir --config $config
+}
+
+function install {
+    rm -erroraction ignore -recurse -force -path $installDir
+    & cmake --install $buildDir --prefix $installDir --config $config
+
+}
+
+init_vars
+git_module_setup
+apply_patches
+build
+install
+
+# TODO - implement ROCm support on windows
+md gguf/build/winrocm/lib -ea 0
+echo $null >> gguf/build/winrocm/lib/.generated

llm/llama.cpp/generate_darwin.go

@@ -0,0 +1,3 @@
+package llm
+
+//go:generate sh ./gen_darwin.sh

llm/llama.cpp/generate_darwin_amd64.go

@@ -1,18 +0,0 @@
-package llm
-
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate git -C ggml apply ../patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch
-//go:generate git -C ggml apply ../patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch
-//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
-//go:generate cmake --build ggml/build/cpu --target server --config Release
-//go:generate mv ggml/build/cpu/bin/server ggml/build/cpu/bin/ollama-runner
-
-//go:generate git submodule update --force gguf
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_METAL=off -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_NAME=Darwin -DCMAKE_SYSTEM_PROCESSOR=x86_64 -DCMAKE_OSX_ARCHITECTURES=x86_64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0 -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=on
-//go:generate cmake --build gguf/build/cpu --target server --config Release
-//go:generate mv gguf/build/cpu/bin/server gguf/build/cpu/bin/ollama-runner

llm/llama.cpp/generate_darwin_arm64.go

@@ -1,18 +0,0 @@
-package llm
-
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate git -C ggml apply ../patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch
-//go:generate git -C ggml apply ../patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch
-//go:generate cmake -S ggml -B ggml/build/metal -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
-//go:generate cmake --build ggml/build/metal --target server --config Release
-//go:generate mv ggml/build/metal/bin/server ggml/build/metal/bin/ollama-runner
-
-//go:generate git submodule update --force gguf
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-//go:generate cmake -S gguf -B gguf/build/metal -DLLAMA_METAL=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DCMAKE_SYSTEM_PROCESSOR=arm64 -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=11.0
-//go:generate cmake --build gguf/build/metal --target server --config Release
-//go:generate mv gguf/build/metal/bin/server gguf/build/metal/bin/ollama-runner

llm/llama.cpp/generate_linux.go

@@ -1,26 +1,3 @@
 package llm
 
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate git -C ggml apply ../patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch
-//go:generate git -C ggml apply ../patches/0001-copy-cuda-runtime-libraries.patch
-//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/cpu --target server --config Release
-//go:generate mv ggml/build/cpu/bin/server ggml/build/cpu/bin/ollama-runner
-
-//go:generate git submodule update --force gguf
-//go:generate git -C gguf apply ../patches/0001-copy-cuda-runtime-libraries.patch
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off
-//go:generate cmake --build gguf/build/cpu --target server --config Release
-//go:generate mv gguf/build/cpu/bin/server gguf/build/cpu/bin/ollama-runner
-
-//go:generate cmake -S ggml -B ggml/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/cuda --target server --config Release
-//go:generate mv ggml/build/cuda/bin/server ggml/build/cuda/bin/ollama-runner
-//go:generate cmake -S gguf -B gguf/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off -DLLAMA_CUDA_PEER_MAX_BATCH_SIZE=0
-//go:generate cmake --build gguf/build/cuda --target server --config Release
-//go:generate mv gguf/build/cuda/bin/server gguf/build/cuda/bin/ollama-runner
+//go:generate bash ./gen_linux.sh

llm/llama.cpp/generate_windows.go

@@ -1,24 +1,3 @@
 package llm
 
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate cmake -S ggml -B ggml/build/cpu -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/cpu --target server --config Release
-//go:generate cmd /c move ggml\build\cpu\bin\Release\server.exe ggml\build\cpu\bin\Release\ollama-runner.exe
-
-//go:generate git submodule update --force gguf
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-//go:generate cmake -S gguf -B gguf/build/cpu -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off
-//go:generate cmake --build gguf/build/cpu --target server --config Release
-//go:generate cmd /c move gguf\build\cpu\bin\Release\server.exe gguf\build\cpu\bin\Release\ollama-runner.exe
-
-//go:generate cmake -S ggml -B ggml/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/cuda --target server --config Release
-//go:generate cmd /c move ggml\build\cuda\bin\Release\server.exe ggml\build\cuda\bin\Release\ollama-runner.exe
-
-//go:generate cmake -S gguf -B gguf/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off
-//go:generate cmake --build gguf/build/cuda --target server --config Release
-//go:generate cmd /c move gguf\build\cuda\bin\Release\server.exe gguf\build\cuda\bin\Release\ollama-runner.exe
+//go:generate powershell -ExecutionPolicy Bypass -File ./gen_windows.ps1

llm/llama.cpp/ggml

@@ -1 +0,0 @@
-Subproject commit 9e232f0234073358e7031c1b8d7aa45020469a3b

llm/llama.cpp/gguf

@@ -1 +1 @@
-Subproject commit 23b5e12eb5a76489b4c3ee22213a081da68b1809
+Subproject commit 9494d7c4774ab745490b5a19570ff7747a194143

llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch

@@ -0,0 +1,550 @@
+From 5671a80d321de622a1702b1cc724505dbaa0fbc3 Mon Sep 17 00:00:00 2001
+From: Bruce MacDonald <brucewmacdonald@gmail.com>
+Date: Thu, 14 Dec 2023 17:09:40 -0500
+Subject: [PATCH] Expose callable API for server
+
+---
+ examples/server/CMakeLists.txt |  24 +++
+ examples/server/server.cpp     | 309 +++++++++++++++++++++++++++++++++
+ examples/server/server.h       |  89 ++++++++++
+ ggml-cuda.cu                   |   1 +
+ llama.cpp                      |  25 +++
+ llama.h                        |   7 +
+ 6 files changed, 455 insertions(+)
+ create mode 100644 examples/server/server.h
+
+diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
+index 859cd12c..4ea47a77 100644
+--- a/examples/server/CMakeLists.txt
++++ b/examples/server/CMakeLists.txt
+@@ -11,3 +11,27 @@ if (WIN32)
+     TARGET_LINK_LIBRARIES(${TARGET} PRIVATE ws2_32)
+ endif()
+ target_compile_features(${TARGET} PRIVATE cxx_std_11)
++
++set(TARGET ext_server)
++option(LLAMA_SERVER_VERBOSE "Build verbose logging option for Server" ON)
++add_library(${TARGET} STATIC server.cpp)
++target_include_directories(${TARGET} PRIVATE ../../common)
++target_include_directories(${TARGET} PRIVATE ../..)
++target_compile_features(${TARGET} PRIVATE cxx_std_11)
++target_compile_definitions(${TARGET} PUBLIC LLAMA_SERVER_LIBRARY=1)
++target_link_libraries(${TARGET} PRIVATE common llama llava ${CMAKE_THREAD_LIBS_INIT})
++
++if (BUILD_SHARED_LIBS)
++    set_target_properties(ext_server PROPERTIES POSITION_INDEPENDENT_CODE ON)
++    target_compile_definitions(ext_server PRIVATE LLAMA_SHARED LLAMA_BUILD)
++    add_library(ext_server_shared SHARED $<TARGET_OBJECTS:ext_server>)
++    target_link_libraries(ext_server_shared PRIVATE ggml llama llava common ${CMAKE_THREAD_LIBS_INIT})
++    install(TARGETS ext_server_shared LIBRARY)
++endif()
++
++if (CUDAToolkit_FOUND)
++    target_include_directories(${TARGET} PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
++    if (WIN32)
++        target_link_libraries(ext_server_shared PRIVATE nvml)
++    endif()
++endif()
+\ No newline at end of file
+diff --git a/examples/server/server.cpp b/examples/server/server.cpp
+index d0cd8e1c..d15a1148 100644
+--- a/examples/server/server.cpp
++++ b/examples/server/server.cpp
+@@ -5,6 +5,9 @@
+ #include "../llava/clip.h"
+ 
+ #include "stb_image.h"
++#if defined(LLAMA_SERVER_LIBRARY)
++#include "server.h"
++#endif
+ 
+ #ifndef NDEBUG
+ // crash the server in debug mode, otherwise send an http 500 error
+@@ -24,6 +27,8 @@
+ #include <thread>
+ #include <mutex>
+ #include <chrono>
++#include <iostream>
++#include <mach/mach.h>
+ 
+ #ifndef SERVER_VERBOSE
+ #define SERVER_VERBOSE 1
+@@ -2632,6 +2637,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
+     }
+ }
+ 
++#ifndef LLAMA_SERVER_LIBRARY
+ int main(int argc, char **argv)
+ {
+     // own arguments required by this example
+@@ -3066,3 +3072,306 @@ int main(int argc, char **argv)
+     llama_backend_free();
+     return 0;
+ }
++
++#else // LLAMA_SERVER_LIBRARY
++// Expose the llama server as a callable extern "C" API
++llama_server_context *llama = NULL;
++std::atomic<bool> ext_server_running(false);
++std::thread ext_server_thread;
++
++static int64_t mem_available() {
++    int mib[2];
++    size_t length;
++    int64_t mem_size;
++
++    mib[0] = CTL_HW;
++    mib[1] = HW_MEMSIZE;
++
++    length = sizeof(mem_size);
++
++    if (sysctl(mib, 2, &mem_size, &length, NULL, 0) != -1) {
++        return mem_size;
++    } else {
++        std::cerr << "Error getting total memory size." << std::endl;
++        return -1;
++    }
++}
++
++void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err)
++{
++    assert(err != NULL && sparams != NULL);
++    err->id = 0;
++    err->msg[0] = '\0';
++
++    try {
++        llama = new llama_server_context;
++        log_set_target(stdout);
++        gpt_params params;
++        params.n_ctx = sparams->n_ctx;
++        params.n_batch = sparams->n_batch;
++        params.n_threads = sparams->n_threads;
++        params.n_parallel = sparams->n_parallel;
++        params.rope_freq_base = sparams->rope_freq_base;
++        params.rope_freq_scale = sparams->rope_freq_scale;
++
++        if (sparams->memory_f16)  {
++            params.cache_type_k = "f16";
++            params.cache_type_v = "f16";
++        } else {
++            params.cache_type_k = "f32";
++            params.cache_type_v = "f32";
++        }
++
++        params.n_gpu_layers = sparams->n_gpu_layers;
++        params.main_gpu = sparams->main_gpu;
++        params.use_mlock = sparams->use_mlock;
++        params.use_mmap = sparams->use_mmap;
++        params.numa = sparams->numa;
++        params.embedding = sparams->embedding;
++        if (sparams->model != NULL) {
++            params.model = sparams->model;
++        }
++
++        for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
++            params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
++        }
++    
++        llama_backend_init(params.numa);
++
++        // check memory requirements
++        // TODO - this is not the right place for this check it should be its own function, but it works for now
++        mem_required mem_req;
++        int64_t mem_ava;
++
++        mem_req = llama_get_mem_required(params.model.c_str(), true); // TODO: check if mmap is set
++        mem_ava = mem_available();
++        LOG_TEE("%s: bruce mem available  = %7.2f MiB\n", __func__, mem_ava / 1024.0 / 1024.0);
++        if (static_cast<int64_t>(mem_req.ctx_size + mem_req.mmapped_size) > mem_ava) {
++            err->id = -1;
++            snprintf(err->msg, err->msg_len, "not enough memory available for model %s, required %ld, available %lld", params.model.c_str(), mem_req.ctx_size + mem_req.mmapped_size, mem_ava);
++            return;
++        }
++
++        // load the model
++        if (!llama->load_model(params))
++        {
++            // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages
++            // and pass them back to the caller for better UX
++            err->id = -1;
++            snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str());
++            return;
++        }
++
++        if (sparams->mmproj != NULL) {
++            params.mmproj = std::string(sparams->mmproj);
++        }
++
++        llama->initialize();
++    } catch (std::exception &e) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
++    } catch (...) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server");
++    }
++}
++
++void llama_server_start()
++{
++    assert(llama != NULL);
++     // TODO mutex to protect thread creation
++    ext_server_thread = std::thread([&]()
++    {
++        ext_server_running = true;
++        try {
++            LOG_TEE("llama server main loop starting\n");
++            ggml_time_init();
++            while (ext_server_running.load())
++            {
++                if (!llama->update_slots()) {
++                    LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n");
++                    break;
++                }
++            }
++        } catch (std::exception &e) {
++            LOG_TEE("caught exception in llama server main loop: %s\n", e.what());
++        } catch (...) {
++            LOG_TEE("caught unknown exception in llama server main loop\n");
++        }
++        LOG_TEE("\nllama server shutting down\n");
++        llama_backend_free();
++    });
++}
++
++void llama_server_stop() {
++    assert(llama != NULL);
++    // TODO - too verbose, remove once things are solid
++    LOG_TEE("requesting llama server shutdown\n");
++    ext_server_running = false;
++    ext_server_thread.join();
++    delete llama;
++    llama = NULL;
++    LOG_TEE("llama server shutdown complete\n");
++}
++
++void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
++    assert(llama != NULL && json_req != NULL && resp != NULL);
++    resp->id = -1;
++    resp->msg[0] = '\0';
++    try {
++        json data = json::parse(json_req);
++        resp->id = llama->request_completion(data, false, false, -1);
++    } catch (std::exception &e) {
++        snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
++    } catch (...) {
++        snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
++    }
++}
++
++void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) {
++    assert(llama != NULL && resp != NULL);
++    std::string msg;
++    resp->id = -1;
++    resp->stop = false;
++    resp->error = false;
++    resp->json_resp = NULL;
++    std::string result_json;
++    try {
++        task_result result = llama->next_result(task_id);
++        result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
++        resp->id = result.id;
++        resp->stop = result.stop;
++        resp->error = result.error;
++        if (result.error) {
++            llama->request_cancel(task_id);
++        } else if (result.stop) {
++            llama->request_cancel(task_id);
++        }
++    } catch (std::exception &e) {
++        resp->error = true;
++        resp->id = -1;
++        result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
++    } catch (...) {
++        resp->error = true;
++        resp->id = -1;
++        result_json = "{\"error\":\"Unknown exception during completion\"}";
++    }
++    const std::string::size_type size = result_json.size() + 1;
++    resp->json_resp = new char[size];
++    snprintf(resp->json_resp, size, "%s", result_json.c_str());
++}
++
++void llama_server_release_task_result(ext_server_task_result_t *result) {
++    if (result == NULL || result->json_resp == NULL) {
++        return;
++    }
++    delete[] result->json_resp;
++}
++
++void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
++    assert(llama != NULL && err != NULL);
++    err->id = 0;
++    err->msg[0] = '\0';
++    try {
++        llama->request_cancel(task_id);
++    } catch (std::exception &e) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
++    } catch (...) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server");
++    }
++}
++
++void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
++    try {
++        const json body = json::parse(json_req);
++        std::vector<llama_token> tokens;
++        if (body.count("content") != 0)
++        {
++            tokens = llama->tokenize(body["content"], false);
++        }
++        const json data = format_tokenizer_response(tokens);
++        std::string result_json = data.dump();
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
++    } catch (std::exception &e) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
++    } catch (...) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
++    }
++}
++
++void llama_server_release_json_resp(char **json_resp) {
++    if (json_resp == NULL || *json_resp == NULL) {
++        return;
++    }
++    delete[] *json_resp;
++}
++
++void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
++    try {
++        const json body = json::parse(json_req);
++        std::string content;
++        if (body.count("tokens") != 0)
++        {
++            const std::vector<llama_token> tokens = body["tokens"];
++            content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
++        }
++        const json data = format_detokenized_response(content);
++        std::string result_json = data.dump();
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
++    } catch (std::exception &e) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
++    } catch (...) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
++    }
++}
++
++void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
++    try {
++        const json body = json::parse(json_req);
++        json prompt;
++        if (body.count("content") != 0)
++        {
++            prompt = body["content"];
++        }
++        else
++        {
++            prompt = "";
++        }
++        const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
++        task_result result = llama->next_result(task_id);
++        std::string result_json = result.result_json.dump();
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
++    } catch (std::exception &e) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
++    } catch (...) {
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
++    }
++}
++
++#endif // LLAMA_SERVER_LIBRARY
+\ No newline at end of file
+diff --git a/examples/server/server.h b/examples/server/server.h
+new file mode 100644
+index 00000000..d22f1b6e
+--- /dev/null
++++ b/examples/server/server.h
+@@ -0,0 +1,89 @@
++#if defined(LLAMA_SERVER_LIBRARY)
++#ifndef LLAMA_SERVER_H
++#define LLAMA_SERVER_H
++#include <stddef.h>
++#include <stdint.h>
++#include <stdio.h>
++#include <stdbool.h>
++
++// This exposes extern C entrypoints into the llama_server 
++// To enable the server compile with LLAMA_SERVER_LIBRARY
++
++#ifdef __cplusplus
++extern "C"
++{
++#endif
++    typedef struct ext_server_resp {
++        int id; // < 0 on error
++        size_t msg_len; // caller must allocate msg and set msg_len
++        char *msg;
++    } ext_server_resp_t;
++
++    // Allocated and freed by caller
++    typedef struct ext_server_lora_adapter {
++        char *adapter;
++        float scale;
++        struct ext_server_lora_adapter *next;
++    } ext_server_lora_adapter_t;
++
++    // Allocated and freed by caller
++    typedef struct ext_server_params
++    {
++        char *model;            
++        uint32_t n_ctx;         // text context, 0 = from model
++        uint32_t n_batch;       // prompt processing maximum batch size
++        uint32_t n_threads;     // number of threads to use for generation
++        int32_t n_parallel;     // number of parallel sequences to decodewra
++        float rope_freq_base;   // RoPE base frequency, 0 = from model
++        float rope_freq_scale;  // RoPE frequency scaling factor, 0 = from model
++        bool memory_f16;        // use f16 instead of f32 for memory kv
++        int32_t n_gpu_layers;   // number of layers to store in VRAM (-1 - use default)
++        int32_t main_gpu;       // the GPU that is used for scratch and small tensors
++        bool use_mlock;         // force system to keep model in RAM
++        bool use_mmap;          // use mmap if possible
++        bool numa;              // attempt optimizations that help on some NUMA systems
++        bool embedding;         // get only sentence embedding
++        ext_server_lora_adapter_t* lora_adapters;
++        char *mmproj;
++    } ext_server_params_t;
++
++    typedef struct ext_server_task_result
++    {
++        int id;
++        bool stop;
++        bool error;
++        char* json_resp; // null terminated, memory managed by ext_server
++    } ext_server_task_result_t;
++
++    // Initialize the server once per process
++    // err->id = 0 for success and err->msg[0] = NULL
++    // err->id != 0 for failure, and err->msg contains error message
++    void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
++
++    // Run the main loop, called once per init
++    void llama_server_start();
++    // Stop the main loop and free up resources allocated in init and start.  Init must be called again to reuse
++    void llama_server_stop();
++
++    // json_req null terminated string, memory managed by caller
++    // resp->id >= 0 on success (task ID)
++    // resp->id < 0 on error, and resp->msg contains error message
++    void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
++
++    // Caller must call llama_server_release_task_result to free resp->json_resp
++    void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result);
++    void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
++    void llama_server_release_task_result(ext_server_task_result_t *result);
++
++    // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0
++    void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
++    void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
++    void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err);
++    void llama_server_release_json_resp(char **json_resp);
++
++#ifdef __cplusplus
++}
++#endif
++
++#endif
++#endif // LLAMA_SERVER_LIBRARY
+\ No newline at end of file
+diff --git a/ggml-cuda.cu b/ggml-cuda.cu
+index 85f7a293..ce51364a 100644
+--- a/ggml-cuda.cu
++++ b/ggml-cuda.cu
+@@ -6410,6 +6410,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
+         CUDA_CHECK(cudaGetDevice(&id));
+         src_ptr = (char *) extra->data_device[id];
+     } else {
++        fprintf(stderr, "ggml_cuda_cpy_tensor_2d assert: backend: %d\n", src->backend);
+         GGML_ASSERT(false);
+     }
+     char * dst_ptr = (char *) dst;
+diff --git a/llama.cpp b/llama.cpp
+index 54fa9e43..e7faca86 100644
+--- a/llama.cpp
++++ b/llama.cpp
+@@ -10172,3 +10172,28 @@ static void llama_log_callback_default(ggml_log_level level, const char * text,
+     fputs(text, stderr);
+     fflush(stderr);
+ }
++
++mem_required llama_get_mem_required(const char * path_model, bool use_mmap) {
++    llama_model* mem_model = new llama_model;
++    mem_required mem_req;
++    try {
++        llama_model_loader ml(path_model, use_mmap, nullptr);
++        
++        llm_load_arch(ml, *mem_model);
++        llm_load_hparams(ml, *mem_model);
++        llm_load_vocab(ml, *mem_model);
++
++        ml.calc_sizes(mem_req.ctx_size, mem_req.mmapped_size);
++
++        LLAMA_LOG_INFO("%s: bruce ctx size      = %7.2f MiB\n", __func__, mem_req.ctx_size / 1024.0 / 1024.0);
++
++        size_t mem_required = mem_req.ctx_size + mem_req.mmapped_size;
++        LLAMA_LOG_INFO("%s: bruce mem required  = %7.2f MiB\n", __func__, mem_required / 1024.0 / 1024.0);
++    } catch (const std::exception& err) {
++        LLAMA_LOG_ERROR("error loading model: %s\n", err.what());
++        delete mem_model;
++        throw; // Rethrow the exception to handle it in the calling code
++    }
++    delete mem_model;
++    return mem_req;
++}
+\ No newline at end of file
+diff --git a/llama.h b/llama.h
+index 45a65cac..d8d0fce5 100644
+--- a/llama.h
++++ b/llama.h
+@@ -849,6 +849,13 @@ extern "C" {
+ 
+     LLAMA_API void llama_dump_timing_info_yaml(FILE * stream, const struct llama_context * ctx);
+ 
++    struct mem_required {
++        size_t ctx_size;
++        size_t mmapped_size;
++    };
++
++    mem_required llama_get_mem_required(const char * path_model, bool use_mmap);
++
+ #ifdef __cplusplus
+ }
+ #endif
+-- 
+2.39.3 (Apple Git-145)
+

llm/llama.cpp/patches/0001-add-detokenize-endpoint.patch

@@ -1,51 +0,0 @@
-From 032ef7ff2423f5117bb59d42fb71be9cebf0a2de Mon Sep 17 00:00:00 2001
-From: Bruce MacDonald <brucewmacdonald@gmail.com>
-Date: Mon, 28 Aug 2023 18:08:12 -0400
-Subject: [PATCH] add detokenize endpoint
-
----
- examples/server/server.cpp | 21 +++++++++++++++++++++
- 1 file changed, 21 insertions(+)
-
-diff --git a/examples/server/server.cpp b/examples/server/server.cpp
-index 9966045..5014691 100644
---- a/examples/server/server.cpp
-+++ b/examples/server/server.cpp
-@@ -1075,6 +1075,12 @@ static json format_tokenizer_response(const std::vector<llama_token> &tokens)
-         {"tokens", tokens}};
- }
- 
-+static json format_detokenized_response(std::string content)
-+{
-+    return json{
-+        {"content", content}};
-+}
-+
- static void parse_options_completion(const json &body, llama_server_context &llama)
- {
-     gpt_params default_params;
-@@ -1361,6 +1367,21 @@ int main(int argc, char **argv)
-         const json data = format_tokenizer_response(tokens);
-         return res.set_content(data.dump(), "application/json"); });
- 
-+    svr.Post("/detokenize", [&llama](const Request &req, Response &res)
-+             {
-+        auto lock = llama.lock();
-+
-+        const json body = json::parse(req.body);
-+        std::string content;
-+        if (body.count("tokens") != 0)
-+        {
-+            const std::vector<llama_token> tokens = body["tokens"];
-+            content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
-+        }
-+
-+        const json data = format_detokenized_response(content);
-+        return res.set_content(data.dump(), "application/json"); });
-+
-     svr.Post("/embedding", [&llama](const Request &req, Response &res)
-              {
-         auto lock = llama.lock();
--- 
-2.39.2 (Apple Git-143)
-

llm/llama.cpp/patches/0001-copy-cuda-runtime-libraries.patch

@@ -1,27 +0,0 @@
-From 5dd02993e8cc2ce309157736b95bb572f274a3fd Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Wed, 20 Sep 2023 14:19:52 -0700
-Subject: [PATCH] copy cuda runtime libraries
-
----
- CMakeLists.txt | 4 ++++
- 1 file changed, 4 insertions(+)
-
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 824d9f2..dd24137 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -274,6 +274,10 @@ if (LLAMA_CUBLAS)
-             set(LLAMA_EXTRA_LIBS ${LLAMA_EXTRA_LIBS} CUDA::cudart CUDA::cublas CUDA::cublasLt)
-         endif()
- 
-+        configure_file(${CUDAToolkit_LIBRARY_DIR}/libcudart.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcudart.so.${CUDAToolkit_VERSION_MAJOR}.0 COPYONLY)
-+        configure_file(${CUDAToolkit_LIBRARY_DIR}/libcublas.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcublas.so.${CUDAToolkit_VERSION_MAJOR} COPYONLY)
-+        configure_file(${CUDAToolkit_LIBRARY_DIR}/libcublasLt.so ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/libcublasLt.so.${CUDAToolkit_VERSION_MAJOR} COPYONLY)
-+
-     if (NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-         # 52 == lowest CUDA 12 standard
-         # 60 == f16 CUDA intrinsics
--- 
-2.42.0
-

llm/llama.cpp/patches/0001-update-default-log-target.patch

@@ -1,25 +0,0 @@
-From 6465fec6290f0a7f5d4d0fbe6bcf634e4810dde6 Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Mon, 23 Oct 2023 10:39:34 -0700
-Subject: [PATCH] default log stderr
-
----
- common/log.h | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/common/log.h b/common/log.h
-index b8953fd..25522cd 100644
---- a/common/log.h
-+++ b/common/log.h
-@@ -90,7 +90,7 @@
- //  }
- //
- #ifndef LOG_TARGET
--    #define LOG_TARGET log_handler()
-+    #define LOG_TARGET nullptr
- #endif
- 
- #ifndef LOG_TEE_TARGET
--- 
-2.42.0
-

llm/llama.cpp/patches/0002-34B-model-support.patch

@@ -1,89 +0,0 @@
-From 6145068a6613c37bb43a7408b5496524bdcfc402 Mon Sep 17 00:00:00 2001
-From: Bruce MacDonald <brucewmacdonald@gmail.com>
-Date: Mon, 28 Aug 2023 18:08:53 -0400
-Subject: [PATCH] 34B model support
-
----
- llama.cpp | 10 ++++++++++
- 1 file changed, 10 insertions(+)
-
-diff --git a/llama.cpp b/llama.cpp
-index f2cbe76..62c5cdf 100644
---- a/llama.cpp
-+++ b/llama.cpp
-@@ -79,6 +79,7 @@ enum e_model {
-     MODEL_7B,
-     MODEL_13B,
-     MODEL_30B,
-+    MODEL_34B,
-     MODEL_65B,
-     MODEL_70B,
- };
-@@ -122,6 +123,7 @@ static std::map<e_model, size_t> MEM_REQ_SCRATCH0(int n_ctx)
-         { MODEL_7B,   ((size_t) n_ctx / 16ull + 100ull) * MB },
-         { MODEL_13B,  ((size_t) n_ctx / 12ull + 120ull) * MB },
-         { MODEL_30B,  ((size_t) n_ctx /  9ull + 160ull) * MB },
-+        { MODEL_34B,  ((size_t) n_ctx / 9ull + 160ull) * MB },
-         { MODEL_65B,  ((size_t) n_ctx /  6ull + 256ull) * MB }, // guess
-         { MODEL_70B,  ((size_t) n_ctx /  7ull + 164ull) * MB },
-     };
-@@ -135,6 +137,7 @@ static const std::map<e_model, size_t> & MEM_REQ_SCRATCH1()
-         { MODEL_7B,  160ull * MB },
-         { MODEL_13B, 192ull * MB },
-         { MODEL_30B, 256ull * MB },
-+        { MODEL_34B, 256ull * MB },
-         { MODEL_65B, 384ull * MB }, // guess
-         { MODEL_70B, 304ull * MB },
-     };
-@@ -149,6 +152,7 @@ static const std::map<e_model, size_t> & MEM_REQ_EVAL()
-         { MODEL_7B,  10ull * MB },
-         { MODEL_13B, 12ull * MB },
-         { MODEL_30B, 16ull * MB },
-+        { MODEL_34B, 16ull * MB },
-         { MODEL_65B, 24ull * MB }, // guess
-         { MODEL_70B, 24ull * MB },
-     };
-@@ -164,6 +168,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_BASE()
-         { MODEL_7B,   512ull * kB },
-         { MODEL_13B,  640ull * kB },
-         { MODEL_30B,  768ull * kB },
-+        { MODEL_34B,  768ull * kB },
-         { MODEL_65B, 1280ull * kB },
-         { MODEL_70B, 1280ull * kB },
-     };
-@@ -179,6 +184,7 @@ static const std::map<e_model, size_t> & VRAM_REQ_SCRATCH_PER_CONTEXT()
-         { MODEL_7B,  128ull },
-         { MODEL_13B, 160ull },
-         { MODEL_30B, 208ull },
-+        { MODEL_34B, 208ull },
-         { MODEL_65B, 256ull },
-         { MODEL_70B, 256ull },
-     };
-@@ -1027,6 +1033,7 @@ static const char * llama_model_type_name(e_model type) {
-         case MODEL_7B: return "7B";
-         case MODEL_13B: return "13B";
-         case MODEL_30B: return "30B";
-+        case MODEL_34B: return "34B";
-         case MODEL_65B: return "65B";
-         case MODEL_70B: return "70B";
-         default: LLAMA_ASSERT(false);
-@@ -1074,6 +1081,7 @@ static void llama_model_load_internal(
-             case 26: model.type = e_model::MODEL_3B; break;
-             case 32: model.type = e_model::MODEL_7B; break;
-             case 40: model.type = e_model::MODEL_13B; break;
-+            case 48: model.type = e_model::MODEL_34B; break;
-             case 60: model.type = e_model::MODEL_30B; break;
-             case 80: model.type = e_model::MODEL_65B; break;
-             default:
-@@ -1094,6 +1102,8 @@ static void llama_model_load_internal(
-             LLAMA_LOG_WARN("%s: warning: assuming 70B model based on GQA == %d\n", __func__, n_gqa);
-             model.type = e_model::MODEL_70B;
-             hparams.f_ffn_mult = 1.3f; // from the params.json of the 70B model
-+        } else if (model.type == e_model::MODEL_34B && n_gqa == 8) {
-+            hparams.f_ffn_mult = 1.0f; // from the params.json of the 34B model
-         }
- 
-         hparams.rope_freq_base  = rope_freq_base;
--- 
-2.39.2 (Apple Git-143)
-

llm/llama.cpp/patches/0003-metal-fix-synchronization-in-new-matrix-multiplicati.patch

@@ -1,30 +0,0 @@
-From dadbed99e65252d79f81101a392d0d6497b86caa Mon Sep 17 00:00:00 2001
-From: Shouzheng Liu <lshzh.hi@gmail.com>
-Date: Mon, 21 Aug 2023 06:59:29 -0400
-Subject: [PATCH] metal : fix synchronization in new matrix multiplication
- kernel (#2686)
-
----
- ggml-metal.metal | 3 ++-
- 1 file changed, 2 insertions(+), 1 deletion(-)
-
-diff --git a/ggml-metal.metal b/ggml-metal.metal
-index 3f31252..88d48f6 100644
---- a/ggml-metal.metal
-+++ b/ggml-metal.metal
-@@ -1898,10 +1898,11 @@ kernel void kernel_mul_mm(device const  uchar * src0,
-         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
-                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
-         for (int i = 0; i < 8; i++) {
-+            threadgroup_barrier(mem_flags::mem_device);
-             simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
-         }
- 
--        threadgroup_barrier(mem_flags::mem_threadgroup);
-+        threadgroup_barrier(mem_flags::mem_device);
-         device float *C = dst + BLOCK_SIZE_M * r0 + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
-         if (sgitg==0) {
-             for (int i = 0; i < n_rows; i++) {
--- 
-2.41.0
-

llm/llama.cpp/patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch

@@ -1,41 +0,0 @@
-From 14b1d7e6f720dee41ce5a826376df738096d9033 Mon Sep 17 00:00:00 2001
-From: Shouzheng Liu <lshzh.hi@gmail.com>
-Date: Tue, 22 Aug 2023 02:18:40 -0400
-Subject: [PATCH] metal : add missing barriers for mul-mat (#2699)
-
----
- ggml-metal.metal | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
-
-diff --git a/ggml-metal.metal b/ggml-metal.metal
-index 88d48f6..ce3541f 100644
---- a/ggml-metal.metal
-+++ b/ggml-metal.metal
-@@ -1850,6 +1850,7 @@ kernel void kernel_mul_mm(device const  uchar * src0,
-         //load data and store to threadgroup memory
-         half4x4 temp_a;
-         dequantize_func(x, il, temp_a);
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         #pragma unroll(16)
-         for (int i = 0; i < 16; i++) {
-             *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
-@@ -1895,14 +1896,14 @@ kernel void kernel_mul_mm(device const  uchar * src0,
-         }
-     } else {
-         // block is smaller than 64x32, we should avoid writing data outside of the matrix
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
-                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
-         for (int i = 0; i < 8; i++) {
--            threadgroup_barrier(mem_flags::mem_device);
-             simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
-         }
- 
--        threadgroup_barrier(mem_flags::mem_device);
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         device float *C = dst + BLOCK_SIZE_M * r0 + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
-         if (sgitg==0) {
-             for (int i = 0; i < n_rows; i++) {
--- 
-2.41.0
-

llm/llama.cpp/patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch

@@ -1,32 +0,0 @@
-From 1e3bc523d8053a77df3ac7126a84d0297ee97ef6 Mon Sep 17 00:00:00 2001
-From: Kylin <56434533+KyL0N@users.noreply.github.com>
-Date: Tue, 22 Aug 2023 15:14:23 +0800
-Subject: [PATCH] ggml : support CUDA's half type for aarch64(#1455) (#2670)
-
-* ggml: support CUDA's half type for aarch64(#1455)
-support CUDA's half type for aarch64 in ggml_fp16_t definition
-
-* ggml: use __CUDACC__ to recognise nvcc compiler
----
- ggml.h | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
-
-diff --git a/ggml.h b/ggml.h
-index 544ad2d..0ec7ec5 100644
---- a/ggml.h
-+++ b/ggml.h
-@@ -259,8 +259,9 @@
- extern "C" {
- #endif
- 
--#ifdef __ARM_NEON
--    // we use the built-in 16-bit float type
-+#if defined(__ARM_NEON) && defined(__CUDACC__)
-+    typedef half ggml_fp16_t;
-+#elif defined(__ARM_NEON)
-     typedef __fp16 ggml_fp16_t;
- #else
-     typedef uint16_t ggml_fp16_t;
--- 
-2.39.2 (Apple Git-143)
-

llm/llama.go

@@ -1,25 +1,13 @@
 package llm
 
 import (
-	"bufio"
 	"bytes"
 	"context"
-	"embed"
-	"encoding/json"
+	_ "embed"
 	"errors"
 	"fmt"
-	"io"
-	"io/fs"
-	"log"
-	"math/rand"
-	"net/http"
 	"os"
 	"os/exec"
-	"path"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -55,109 +43,6 @@ number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
 ws ::= ([ \t\n] ws)?
 `
 
-//go:embed llama.cpp/*/build/*/bin/*
-var llamaCppEmbed embed.FS
-
-type ModelRunner struct {
-	Type        string // "gguf" or "ggml"
-	Path        string // path to the model runner executable
-	Accelerated bool
-}
-
-func chooseRunners(workDir, runnerType string) []ModelRunner {
-	buildPath := path.Join("llama.cpp", runnerType, "build")
-	var runners []ModelRunner
-
-	// set the runners based on the OS
-	// IMPORTANT: the order of the runners in the array is the priority order
-	switch runtime.GOOS {
-	case "darwin":
-		if runtime.GOARCH == "arm64" {
-			runners = []ModelRunner{{Type: runnerType, Path: path.Join(buildPath, "metal", "bin", "ollama-runner")}}
-		} else {
-			runners = []ModelRunner{{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")}}
-		}
-	case "linux":
-		runners = []ModelRunner{
-			{Type: runnerType, Path: path.Join(buildPath, "cuda", "bin", "ollama-runner"), Accelerated: true},
-			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
-		}
-	case "windows":
-		// TODO: select windows GPU runner here when available
-		runners = []ModelRunner{
-			{Type: runnerType, Path: path.Join(buildPath, "cuda", "bin", "Release", "ollama-runner.exe"), Accelerated: true},
-			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "Release", "ollama-runner.exe")},
-		}
-	default:
-		log.Printf("unknown OS, running on CPU: %s", runtime.GOOS)
-		runners = []ModelRunner{
-			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
-		}
-	}
-
-	runnerAvailable := false // if no runner files are found in the embed, this flag will cause a fast fail
-	for _, r := range runners {
-		// find all the files in the runner's bin directory
-		files, err := fs.Glob(llamaCppEmbed, path.Join(path.Dir(r.Path), "*"))
-		if err != nil {
-			// this is expected, ollama may be compiled without all runners packed in
-			log.Printf("%s runner not found: %v", r.Path, err)
-			continue
-		}
-
-		for _, f := range files {
-			runnerAvailable = true
-
-			srcFile, err := llamaCppEmbed.Open(f)
-			if err != nil {
-				log.Fatalf("read llama runner %s: %v", f, err)
-			}
-			defer srcFile.Close()
-
-			// create the directory in case it does not exist, filepath.Dir() converts the file path to the OS's format
-			destPath := filepath.Join(workDir, filepath.Dir(f))
-			if err := os.MkdirAll(destPath, 0o755); err != nil {
-				log.Fatalf("create runner temp dir %s: %v", filepath.Dir(f), err)
-			}
-
-			// create the path to the destination file, filepath.Base() converts the file path to the OS's format
-			destFile := filepath.Join(destPath, filepath.Base(f))
-
-			_, err = os.Stat(destFile)
-			switch {
-			case errors.Is(err, os.ErrNotExist):
-				destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
-				if err != nil {
-					log.Fatalf("write llama runner %s: %v", f, err)
-				}
-				defer destFile.Close()
-
-				if _, err := io.Copy(destFile, srcFile); err != nil {
-					log.Fatalf("copy llama runner %s: %v", f, err)
-				}
-			case err != nil:
-				log.Fatalf("stat llama runner %s: %v", f, err)
-			}
-		}
-	}
-	if !runnerAvailable {
-		log.Fatalf("%s runner not found", runnerType)
-	}
-
-	// return the runners to try in priority order
-	localRunnersByPriority := []ModelRunner{}
-	for _, r := range runners {
-		// clean the ModelRunner paths so that they match the OS we are running on
-		localRunnersByPriority = append(localRunnersByPriority, ModelRunner{
-			Type:        r.Type,
-			Path:        filepath.Clean(path.Join(workDir, r.Path)),
-			Accelerated: r.Accelerated,
-		})
-	}
-
-	return localRunnersByPriority
-}
-
 type llamaModel struct {
 	hyperparameters llamaHyperparameters
 }
@@ -239,72 +124,6 @@ var (
 	errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only")
 )
 
-// CheckVRAM returns the free VRAM in bytes on Linux machines with NVIDIA GPUs
-func CheckVRAM() (int64, error) {
-	cmd := exec.Command("nvidia-smi", "--query-gpu=memory.free", "--format=csv,noheader,nounits")
-	var stdout bytes.Buffer
-	cmd.Stdout = &stdout
-	err := cmd.Run()
-	if err != nil {
-		return 0, errNvidiaSMI
-	}
-
-	var freeMiB int64
-	scanner := bufio.NewScanner(&stdout)
-	for scanner.Scan() {
-		line := scanner.Text()
-		if strings.Contains(line, "[Insufficient Permissions]") {
-			return 0, fmt.Errorf("GPU support may not enabled, check you have installed GPU drivers and have the necessary permissions to run nvidia-smi")
-		}
-
-		vram, err := strconv.ParseInt(strings.TrimSpace(line), 10, 64)
-		if err != nil {
-			return 0, fmt.Errorf("failed to parse available VRAM: %v", err)
-		}
-
-		freeMiB += vram
-	}
-
-	freeBytes := freeMiB * 1024 * 1024
-	if freeBytes < 2*format.GigaByte {
-		log.Printf("less than 2 GB VRAM available")
-		return 0, errAvailableVRAM
-	}
-
-	return freeBytes, nil
-}
-
-func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
-	if opts.NumGPU != -1 {
-		return opts.NumGPU
-	}
-	if runtime.GOOS == "linux" || runtime.GOOS == "windows" {
-		freeBytes, err := CheckVRAM()
-		if err != nil {
-			if !errors.Is(err, errNvidiaSMI) {
-				log.Print(err.Error())
-			}
-			// nvidia driver not installed or no nvidia GPU found
-			return 0
-		}
-
-		/*
-		 Calculate bytes per layer, this will roughly be the size of the model file divided by the number of layers.
-		 We can store the model weights and the kv cache in vram,
-		 to enable kv chache vram storage add two additional layers to the number of layers retrieved from the model file.
-		*/
-		bytesPerLayer := fileSizeBytes / numLayer
-
-		// 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors
-		layers := int(freeBytes/bytesPerLayer) * 3 / 4
-		log.Printf("%d MB VRAM available, loading up to %d GPU layers", freeBytes/(1024*1024), layers)
-
-		return layers
-	}
-	// default to enable metal on macOS
-	return 1
-}
-
 // StatusWriter is a writer that captures error messages from the llama runner process
 type StatusWriter struct {
 	ErrCh      chan error
@@ -333,203 +152,6 @@ func (w *StatusWriter) Write(b []byte) (int, error) {
 	return os.Stderr.Write(b)
 }
 
-func newLlama(model string, adapters, projectors []string, runners []ModelRunner, numLayers int64, opts api.Options) (*llama, error) {
-	fileInfo, err := os.Stat(model)
-	if err != nil {
-		return nil, err
-	}
-
-	if len(adapters) > 1 {
-		return nil, errors.New("ollama supports only one lora adapter, but multiple were provided")
-	}
-
-	numGPU := NumGPU(numLayers, fileInfo.Size(), opts)
-	params := []string{
-		"--model", model,
-		"--ctx-size", fmt.Sprintf("%d", opts.NumCtx),
-		"--batch-size", fmt.Sprintf("%d", opts.NumBatch),
-		"--n-gpu-layers", fmt.Sprintf("%d", numGPU),
-		"--embedding",
-	}
-
-	if opts.MainGPU > 0 {
-		params = append(params, "--main-gpu", fmt.Sprintf("%d", opts.MainGPU))
-	}
-
-	if opts.RopeFrequencyBase > 0 {
-		params = append(params, "--rope-freq-base", fmt.Sprintf("%f", opts.RopeFrequencyBase))
-	}
-
-	if opts.RopeFrequencyScale > 0 {
-		params = append(params, "--rope-freq-scale", fmt.Sprintf("%f", opts.RopeFrequencyScale))
-	}
-
-	if opts.NumGQA > 0 {
-		params = append(params, "--gqa", fmt.Sprintf("%d", opts.NumGQA))
-	}
-
-	if len(adapters) > 0 {
-		// TODO: applying multiple adapters is not supported by the llama.cpp server yet
-		params = append(params, "--lora", adapters[0])
-	}
-
-	if len(projectors) > 0 {
-		// TODO: applying multiple projectors is not supported by the llama.cpp server yet
-		params = append(params, "--mmproj", projectors[0])
-	}
-
-	if opts.NumThread > 0 {
-		params = append(params, "--threads", fmt.Sprintf("%d", opts.NumThread))
-	}
-
-	if !opts.F16KV {
-		params = append(params, "--memory-f32")
-	}
-	if opts.UseMLock {
-		params = append(params, "--mlock")
-	}
-	if !opts.UseMMap {
-		params = append(params, "--no-mmap")
-	}
-	if opts.UseNUMA {
-		params = append(params, "--numa")
-	}
-
-	var runnerErr error
-
-	// start the llama.cpp server with a retry in case the port is already in use
-	for _, runner := range runners {
-		if runner.Accelerated && numGPU == 0 {
-			log.Printf("skipping accelerated runner because num_gpu=0")
-			continue
-		}
-
-		if _, err := os.Stat(runner.Path); err != nil {
-			log.Printf("llama runner not found: %v", err)
-			continue
-		}
-
-		port := rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
-		params := append(params, "--port", strconv.Itoa(port))
-
-		ctx, cancel := context.WithCancel(context.Background())
-		cmd := exec.CommandContext(
-			ctx,
-			runner.Path,
-			params...,
-		)
-
-		var libraryPaths []string
-		if libraryPath, ok := os.LookupEnv("LD_LIBRARY_PATH"); ok {
-			libraryPaths = append(libraryPaths, libraryPath)
-		}
-
-		libraryPaths = append(libraryPaths, filepath.Dir(runner.Path))
-
-		cmd.Env = append(os.Environ(), fmt.Sprintf("LD_LIBRARY_PATH=%s", strings.Join(libraryPaths, ":")))
-		cmd.Stdout = os.Stderr
-		statusWriter := NewStatusWriter()
-		cmd.Stderr = statusWriter
-
-		llm := &llama{Options: opts, Running: Running{Port: port, Cmd: cmd, Cancel: cancel, exitCh: make(chan error)}}
-
-		log.Print("starting llama runner")
-		if err := llm.Cmd.Start(); err != nil {
-			log.Printf("error starting the external llama runner: %v", err)
-			continue
-		}
-
-		// monitor the llama runner process and signal when it exits
-		go func() {
-			err := llm.Cmd.Wait()
-			// default to printing the exit message of the command process, it will probably just say 'exit staus 1'
-			errMsg := err.Error()
-			// try to set a better error message if llama runner logs captured an error
-			if statusWriter.LastErrMsg != "" {
-				errMsg = statusWriter.LastErrMsg
-			}
-			log.Println(errMsg)
-			// llm.Cmd.Wait() can only be called once, use this exit channel to signal that the process has exited
-			llm.exitOnce.Do(func() {
-				close(llm.exitCh)
-			})
-		}()
-
-		if err := waitForServer(llm); err != nil {
-			log.Printf("error starting llama runner: %v", err)
-			llm.Close()
-
-			// default the runnerErr to the error returned by the most recent llama runner process
-			runnerErr = err
-
-			// capture the error directly from the runner process, if any
-			select {
-			case runnerErr = <-statusWriter.ErrCh:
-			default:
-				// the runner process probably timed out
-			}
-
-			// try again
-			continue
-		}
-
-		// server started successfully
-		return llm, nil
-	}
-
-	if runnerErr != nil {
-		// this is the error returned from the llama runner process that failed most recently
-		return nil, runnerErr
-	}
-
-	return nil, fmt.Errorf("failed to start a llama runner")
-}
-
-func waitForServer(llm *llama) error {
-	start := time.Now()
-	expiresAt := time.Now().Add(3 * time.Minute) // be generous with timeout, large models can take a while to load
-	ticker := time.NewTicker(200 * time.Millisecond)
-	defer ticker.Stop()
-
-	log.Print("waiting for llama runner to start responding")
-	for {
-		select {
-		case <-llm.exitCh:
-			// failed to start subprocess
-			return fmt.Errorf("llama runner process has terminated")
-		case <-ticker.C:
-			if time.Now().After(expiresAt) {
-				// timeout
-				return fmt.Errorf("timed out waiting for llama runner to start")
-			}
-
-			if err := llm.Ping(context.Background()); err == nil {
-				// success
-				log.Printf("llama runner started in %f seconds", time.Since(start).Seconds())
-				return nil
-			}
-		}
-	}
-}
-
-func (llm *llama) Close() {
-	// signal the sub-process to terminate
-	llm.Cancel()
-
-	// wait for the command to exit to prevent race conditions with the next run
-	<-llm.exitCh
-
-	if llm.StatusWriter != nil && llm.StatusWriter.LastErrMsg != "" {
-		log.Printf("llama runner stopped with error: %v", llm.StatusWriter.LastErrMsg)
-	} else {
-		log.Print("llama runner stopped successfully")
-	}
-}
-
-func (llm *llama) SetOptions(opts api.Options) {
-	llm.Options = opts
-}
-
 type prediction struct {
 	Content string `json:"content"`
 	Model   string `json:"model"`
@@ -567,162 +189,6 @@ type PredictResult struct {
 	EvalDuration       time.Duration
 }
 
-// IsRetryable checks if the line matches a condition that can be retried
-func isRetryable(line []byte) bool {
-	return bytes.Contains(line, []byte("slot unavailable"))
-}
-
-func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(PredictResult)) error {
-	imageData := llm.ImageData
-	if len(predict.Images) > 0 {
-		for cnt, i := range predict.Images {
-			imageData = append(imageData, ImageData{Data: i, ID: cnt})
-		}
-	}
-	log.Printf("loaded %d images", len(imageData))
-
-	request := map[string]any{
-		"prompt":            predict.Prompt,
-		"stream":            true,
-		"n_predict":         llm.NumPredict,
-		"n_keep":            llm.NumKeep,
-		"main_gpu":          llm.MainGPU,
-		"temperature":       llm.Temperature,
-		"top_k":             llm.TopK,
-		"top_p":             llm.TopP,
-		"tfs_z":             llm.TFSZ,
-		"typical_p":         llm.TypicalP,
-		"repeat_last_n":     llm.RepeatLastN,
-		"repeat_penalty":    llm.RepeatPenalty,
-		"presence_penalty":  llm.PresencePenalty,
-		"frequency_penalty": llm.FrequencyPenalty,
-		"mirostat":          llm.Mirostat,
-		"mirostat_tau":      llm.MirostatTau,
-		"mirostat_eta":      llm.MirostatEta,
-		"penalize_nl":       llm.PenalizeNewline,
-		"seed":              llm.Seed,
-		"stop":              llm.Stop,
-		"image_data":        imageData,
-	}
-
-	if predict.Format == "json" {
-		request["grammar"] = jsonGrammar
-	}
-
-	retryDelay := 100 * time.Microsecond
-	for retries := 0; retries < maxRetries; retries++ {
-		if retries > 0 {
-			time.Sleep(retryDelay) // wait before retrying
-			retryDelay *= 2        // exponential backoff
-		}
-
-		// Handling JSON marshaling with special characters unescaped.
-		buffer := &bytes.Buffer{}
-		enc := json.NewEncoder(buffer)
-		enc.SetEscapeHTML(false)
-
-		if err := enc.Encode(request); err != nil {
-			return fmt.Errorf("failed to marshal data: %v", err)
-		}
-
-		endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", llm.Port)
-		req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, buffer)
-		if err != nil {
-			return fmt.Errorf("error creating POST request: %v", err)
-		}
-		req.Header.Set("Content-Type", "application/json")
-
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return fmt.Errorf("POST predict: %v", err)
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode >= 400 {
-			bodyBytes, err := io.ReadAll(resp.Body)
-			if err != nil {
-				return fmt.Errorf("failed reading llm error response: %w", err)
-			}
-			log.Printf("llm predict error: %s", bodyBytes)
-			return fmt.Errorf("%s", bodyBytes)
-		}
-
-		scanner := bufio.NewScanner(resp.Body)
-		// increase the buffer size to avoid running out of space
-		buf := make([]byte, 0, maxBufferSize)
-		scanner.Buffer(buf, maxBufferSize)
-
-		retryNeeded := false
-		for scanner.Scan() {
-			select {
-			case <-ctx.Done():
-				// This handles the request cancellation
-				return ctx.Err()
-			default:
-				line := scanner.Bytes()
-				if len(line) == 0 {
-					continue
-				}
-
-				if isRetryable(line) {
-					retryNeeded = true
-					break
-				}
-
-				evt, ok := bytes.CutPrefix(line, []byte("data: "))
-				if !ok {
-					return fmt.Errorf("error parsing llm response stream: %s", line)
-				}
-
-				var p prediction
-				if err := json.Unmarshal(evt, &p); err != nil {
-					return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
-				}
-
-				if p.Content != "" {
-					fn(PredictResult{
-						CreatedAt: time.Now().UTC(),
-						Content:   p.Content,
-					})
-				}
-
-				if p.Stop {
-					fn(PredictResult{
-						CreatedAt:     time.Now().UTC(),
-						TotalDuration: time.Since(predict.CheckpointStart),
-
-						Done:               true,
-						PromptEvalCount:    p.Timings.PromptN,
-						PromptEvalDuration: parseDurationMs(p.Timings.PromptMS),
-						EvalCount:          p.Timings.PredictedN,
-						EvalDuration:       parseDurationMs(p.Timings.PredictedMS),
-					})
-					return nil
-				}
-			}
-		}
-
-		if err := scanner.Err(); err != nil {
-			if strings.Contains(err.Error(), "unexpected EOF") {
-				// this means the llama runner subprocess crashed
-				llm.Close()
-				if llm.StatusWriter != nil && llm.StatusWriter.LastErrMsg != "" {
-					return fmt.Errorf("llama runner exited: %v", llm.StatusWriter.LastErrMsg)
-				}
-				return fmt.Errorf("llama runner exited, you may not have enough available memory to run this model")
-			}
-			return fmt.Errorf("error reading llm response: %v", err)
-		}
-
-		if !retryNeeded {
-			return nil // success
-		}
-	}
-
-	// should never reach here ideally
-	return fmt.Errorf("max retries exceeded")
-}
-
 type TokenizeRequest struct {
 	Content string `json:"content"`
 }
@@ -731,43 +197,6 @@ type TokenizeResponse struct {
 	Tokens []int `json:"tokens"`
 }
 
-func (llm *llama) Encode(ctx context.Context, prompt string) ([]int, error) {
-	endpoint := fmt.Sprintf("http://127.0.0.1:%d/tokenize", llm.Port)
-	data, err := json.Marshal(TokenizeRequest{Content: prompt})
-	if err != nil {
-		return nil, fmt.Errorf("marshaling encode data: %w", err)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
-	if err != nil {
-		return nil, fmt.Errorf("encode request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("do encode request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("read encode request: %w", err)
-	}
-
-	if resp.StatusCode >= 400 {
-		log.Printf("llm encode error: %s", body)
-		return nil, fmt.Errorf("%s", body)
-	}
-
-	var encoded TokenizeResponse
-	if err := json.Unmarshal(body, &encoded); err != nil {
-		return nil, fmt.Errorf("unmarshal encode response: %w", err)
-	}
-
-	return encoded.Tokens, nil
-}
-
 type DetokenizeRequest struct {
 	Tokens []int `json:"tokens"`
 }
@@ -776,46 +205,6 @@ type DetokenizeResponse struct {
 	Content string `json:"content"`
 }
 
-func (llm *llama) Decode(ctx context.Context, tokens []int) (string, error) {
-	if len(tokens) == 0 {
-		return "", nil
-	}
-	endpoint := fmt.Sprintf("http://127.0.0.1:%d/detokenize", llm.Port)
-	data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
-	if err != nil {
-		return "", fmt.Errorf("marshaling decode data: %w", err)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
-	if err != nil {
-		return "", fmt.Errorf("decode request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("do decode request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", fmt.Errorf("read decode request: %w", err)
-	}
-
-	if resp.StatusCode >= 400 {
-		log.Printf("llm decode error: %s", body)
-		return "", fmt.Errorf("%s", body)
-	}
-
-	var decoded DetokenizeResponse
-	if err := json.Unmarshal(body, &decoded); err != nil {
-		return "", fmt.Errorf("unmarshal encode response: %w", err)
-	}
-
-	return decoded.Content, nil
-}
-
 type EmbeddingRequest struct {
 	Content string `json:"content"`
 }
@@ -823,52 +212,3 @@ type EmbeddingRequest struct {
 type EmbeddingResponse struct {
 	Embedding []float64 `json:"embedding"`
 }
-
-func (llm *llama) Embedding(ctx context.Context, input string) ([]float64, error) {
-	endpoint := fmt.Sprintf("http://127.0.0.1:%d/embedding", llm.Port)
-	data, err := json.Marshal(TokenizeRequest{Content: input})
-	if err != nil {
-		return nil, fmt.Errorf("error marshaling embed data: %w", err)
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewBuffer(data))
-	if err != nil {
-		return nil, fmt.Errorf("error creating embed request: %w", err)
-	}
-	req.Header.Set("Content-Type", "application/json")
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return nil, fmt.Errorf("POST embedding: %w", err)
-	}
-	defer resp.Body.Close()
-
-	body, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return nil, fmt.Errorf("error reading embed response: %w", err)
-	}
-
-	if resp.StatusCode >= 400 {
-		log.Printf("llm encode error: %s", body)
-		return nil, fmt.Errorf("%s", body)
-	}
-
-	var embedding EmbeddingResponse
-	if err := json.Unmarshal(body, &embedding); err != nil {
-		return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
-	}
-
-	return embedding.Embedding, nil
-}
-
-// Ping checks that the server subprocess is still running and responding to requests
-func (llm *llama) Ping(ctx context.Context) error {
-	resp, err := http.Head(fmt.Sprintf("http://127.0.0.1:%d", llm.Port))
-	if err != nil {
-		return fmt.Errorf("ping resp: %w", err)
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("unexpected ping status: %s", resp.Status)
-	}
-	return nil
-}

llm/llm.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/format"
+	"github.com/jmorganca/ollama/gpu"
 )
 
 type LLM interface {
@@ -18,7 +19,6 @@ type LLM interface {
 	Embedding(context.Context, string) ([]float64, error)
 	Encode(context.Context, string) ([]int, error)
 	Decode(context.Context, []int) (string, error)
-	SetOptions(api.Options)
 	Close()
 	Ping(context.Context) error
 }
@@ -76,16 +76,19 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
 		}
 	}
 
-	switch ggml.Name() {
-	case "gguf":
-		// TODO: gguf will load these options automatically from the model binary
-		opts.NumGQA = 0
-		opts.RopeFrequencyBase = 0.0
-		opts.RopeFrequencyScale = 0.0
-		return newLlama(model, adapters, projectors, chooseRunners(workDir, "gguf"), ggml.NumLayers(), opts)
-	case "ggml", "ggmf", "ggjt", "ggla":
-		return newLlama(model, adapters, projectors, chooseRunners(workDir, "ggml"), ggml.NumLayers(), opts)
+	opts.NumGQA = 0
+	opts.RopeFrequencyBase = 0.0
+	opts.RopeFrequencyScale = 0.0
+	gpuInfo := gpu.GetGPUInfo()
+	switch gpuInfo.Driver {
+	case "ROCM":
+		return newRocmShimExtServer(model, adapters, projectors, ggml.NumLayers(), opts)
 	default:
-		return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
+		// Rely on the built-in CUDA based server which will fall back to CPU
+		return newLlamaExtServer(model, adapters, projectors, ggml.NumLayers(), opts)
 	}
 }
+
+func Init(workdir string) error {
+	return nativeInit(workdir)
+}

llm/rocm_shim.c

@@ -0,0 +1,134 @@
+#include "rocm_shim.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <dlfcn.h>
+#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
+#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
+#define LOAD_ERR() dlerror()
+#define UNLOAD_LIBRARY(handle) dlclose(handle)
+#else
+#include <windows.h>
+#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
+#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
+#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
+// TODO - threadsafety...
+inline static char *LOAD_ERR() {
+  static char errbuf[8];
+  snprintf(errbuf, 8, "0x%lx", GetLastError());
+  return errbuf;
+}
+#endif
+
+void rocm_shim_init(const char *libPath, struct rocm_llama_server *s,
+                    ext_server_resp_t *err) {
+  int i = 0;
+  struct lookup {
+    char *s;
+    void **p;
+  } l[] = {
+      {"llama_server_init", (void *)&s->llama_server_init},
+      {"llama_server_start", (void *)&s->llama_server_start},
+      {"llama_server_stop", (void *)&s->llama_server_stop},
+      {"llama_server_completion", (void *)&s->llama_server_completion},
+      {"llama_server_completion_next_result",
+       (void *)&s->llama_server_completion_next_result},
+      {"llama_server_completion_cancel",
+       (void *)&s->llama_server_completion_cancel},
+      {"llama_server_release_task_result",
+       (void *)&s->llama_server_release_task_result},
+      {"llama_server_tokenize", (void *)&s->llama_server_tokenize},
+      {"llama_server_detokenize", (void *)&s->llama_server_detokenize},
+      {"llama_server_embedding", (void *)&s->llama_server_embedding},
+      {"llama_server_release_json_resp",
+       (void *)&s->llama_server_release_json_resp},
+      {"", NULL},
+  };
+
+  printf("Lazy loading %s library\n", libPath);
+  s->handle = LOAD_LIBRARY(libPath, RTLD_LAZY);
+  if (!s->handle) {
+    err->id = -1;
+    snprintf(
+        err->msg, err->msg_len,
+        "Unable to load rocm server library: %s (If you have a Radeon card, "
+        "did you install the ROCM libraries?)",
+        LOAD_ERR());
+    return;
+  }
+
+  for (i = 0; l[i].p != NULL; i++) {
+    *l[i].p = LOAD_SYMBOL(s->handle, l[i].s);
+    if (!l[i].p) {
+      UNLOAD_LIBRARY(s->handle);
+      err->id = -1;
+      snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s",
+               l[i].s, LOAD_ERR());
+      return;
+    }
+  }
+}
+
+inline void rocm_shim_llama_server_init(struct rocm_llama_server s,
+                                        ext_server_params_t *sparams,
+                                        ext_server_resp_t *err) {
+  s.llama_server_init(sparams, err);
+}
+
+inline void rocm_shim_llama_server_start(struct rocm_llama_server s) {
+  s.llama_server_start();
+}
+
+inline void rocm_shim_llama_server_stop(struct rocm_llama_server s) {
+  s.llama_server_stop();
+}
+
+inline void rocm_shim_llama_server_completion(struct rocm_llama_server s,
+                                              const char *json_req,
+                                              ext_server_resp_t *resp) {
+  s.llama_server_completion(json_req, resp);
+}
+
+inline void rocm_shim_llama_server_completion_next_result(
+    struct rocm_llama_server s, const int task_id,
+    ext_server_task_result_t *result) {
+  s.llama_server_completion_next_result(task_id, result);
+}
+
+inline void rocm_shim_llama_server_completion_cancel(struct rocm_llama_server s,
+                                                     const int task_id,
+                                                     ext_server_resp_t *err) {
+  s.llama_server_completion_cancel(task_id, err);
+}
+inline void rocm_shim_llama_server_release_task_result(
+    struct rocm_llama_server s, ext_server_task_result_t *result) {
+  s.llama_server_release_task_result(result);
+}
+
+inline void rocm_shim_llama_server_tokenize(struct rocm_llama_server s,
+                                            const char *json_req,
+                                            char **json_resp,
+                                            ext_server_resp_t *err) {
+  s.llama_server_tokenize(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_detokenize(struct rocm_llama_server s,
+                                              const char *json_req,
+                                              char **json_resp,
+                                              ext_server_resp_t *err) {
+  s.llama_server_detokenize(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_embedding(struct rocm_llama_server s,
+                                             const char *json_req,
+                                             char **json_resp,
+                                             ext_server_resp_t *err) {
+  s.llama_server_embedding(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_release_json_resp(struct rocm_llama_server s,
+                                                     char **json_resp) {
+  s.llama_server_release_json_resp(json_resp);
+}

llm/rocm_shim.h

@@ -0,0 +1,73 @@
+#include <stdlib.h>
+
+#include "server.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+struct rocm_llama_server {
+  void *handle;
+  void (*llama_server_init)(ext_server_params_t *sparams,
+                            ext_server_resp_t *err);
+  void (*llama_server_start)();
+  void (*llama_server_stop)();
+  void (*llama_server_completion)(const char *json_req,
+                                  ext_server_resp_t *resp);
+  void (*llama_server_completion_next_result)(const int task_id,
+                                              ext_server_task_result_t *result);
+  void (*llama_server_completion_cancel)(const int task_id,
+                                         ext_server_resp_t *err);
+  void (*llama_server_release_task_result)(ext_server_task_result_t *result);
+  void (*llama_server_tokenize)(const char *json_req, char **json_resp,
+                                ext_server_resp_t *err);
+  void (*llama_server_detokenize)(const char *json_req, char **json_resp,
+                                  ext_server_resp_t *err);
+  void (*llama_server_embedding)(const char *json_req, char **json_resp,
+                                 ext_server_resp_t *err);
+  void (*llama_server_release_json_resp)(char **json_resp);
+};
+
+void rocm_shim_init(const char *libPath, struct rocm_llama_server *s,
+                    ext_server_resp_t *err);
+
+// No good way to call C function pointers from Go so inline the indirection
+void rocm_shim_llama_server_init(struct rocm_llama_server s,
+                                 ext_server_params_t *sparams,
+                                 ext_server_resp_t *err);
+
+void rocm_shim_llama_server_start(struct rocm_llama_server s);
+
+void rocm_shim_llama_server_stop(struct rocm_llama_server s);
+
+void rocm_shim_llama_server_completion(struct rocm_llama_server s,
+                                       const char *json_req,
+                                       ext_server_resp_t *resp);
+
+void rocm_shim_llama_server_completion_next_result(
+    struct rocm_llama_server s, const int task_id,
+    ext_server_task_result_t *result);
+
+void rocm_shim_llama_server_completion_cancel(struct rocm_llama_server s,
+                                              const int task_id,
+                                              ext_server_resp_t *err);
+
+void rocm_shim_llama_server_release_task_result(
+    struct rocm_llama_server s, ext_server_task_result_t *result);
+
+void rocm_shim_llama_server_tokenize(struct rocm_llama_server s,
+                                     const char *json_req, char **json_resp,
+                                     ext_server_resp_t *err);
+
+void rocm_shim_llama_server_detokenize(struct rocm_llama_server s,
+                                       const char *json_req, char **json_resp,
+                                       ext_server_resp_t *err);
+
+void rocm_shim_llama_server_embedding(struct rocm_llama_server s,
+                                      const char *json_req, char **json_resp,
+                                      ext_server_resp_t *err);
+void rocm_shim_llama_server_release_json_resp(struct rocm_llama_server s,
+                                              char **json_resp);
+
+#ifdef __cplusplus
+}
+#endif

llm/shim_darwin.go

@@ -0,0 +1,18 @@
+package llm
+
+import (
+	"fmt"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+// no-op stubs for mac
+
+func newRocmShimExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	// should never happen...
+	return nil, fmt.Errorf("ROCM GPUs not supported on Mac")
+}
+
+func nativeInit(workDir string) error {
+	return nil
+}

llm/shim_ext_server.go

@@ -0,0 +1,215 @@
+//go:build !darwin
+
+package llm
+
+/*
+
+#include <stdlib.h>
+#include "rocm_shim.h"
+
+*/
+import "C"
+import (
+	"context"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"unsafe"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+//go:embed llama.cpp/gguf/build/*/lib/*
+var libEmbed embed.FS
+
+var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
+var NoShim = true
+
+type shimExtServer struct {
+	s       C.struct_rocm_llama_server
+	options api.Options
+}
+
+// Note: current implementation does not support concurrent instantiations
+var shimMutex sync.Mutex
+var llm *shimExtServer
+
+func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_init(llm.s, sparams, err)
+}
+func (llm *shimExtServer) llama_server_start() {
+	C.rocm_shim_llama_server_start(llm.s)
+}
+func (llm *shimExtServer) llama_server_stop() {
+	C.rocm_shim_llama_server_stop(llm.s)
+}
+
+func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_completion(llm.s, json_req, resp)
+}
+func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
+	C.rocm_shim_llama_server_completion_next_result(llm.s, task_id, resp)
+}
+func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_completion_cancel(llm.s, task_id, err)
+}
+func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
+	C.rocm_shim_llama_server_release_task_result(llm.s, result)
+}
+
+func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
+	C.rocm_shim_llama_server_release_json_resp(llm.s, json_resp)
+}
+
+func newRocmShimExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	if NoShim {
+		return nil, RocmShimMissing
+	}
+	log.Printf("Loading ROCM llm server")
+	if llm == nil {
+		return nil, fmt.Errorf("nativeInit wasnt called or libary load failed")
+	}
+	llm.options = opts
+	return newExtServer(llm, model, adapters, projectors, numLayers, opts)
+}
+
+func (llm *shimExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
+	return predict(llm, llm.options, ctx, pred, fn)
+}
+
+func (llm *shimExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
+	return encode(llm, ctx, prompt)
+}
+
+func (llm *shimExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
+	return decode(llm, ctx, tokens)
+}
+
+func (llm *shimExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
+	return embedding(llm, ctx, input)
+}
+
+func (llm *shimExtServer) Ping(ctx context.Context) error {
+	return nil
+}
+
+func (llm *shimExtServer) Close() {
+	close(llm)
+}
+
+func nativeInit(workdir string) error {
+	err := extractLib(workdir)
+	if err != nil {
+		if err == RocmShimMissing {
+			log.Printf("%s", err)
+			return nil
+		}
+		return err
+	}
+
+	// Verify we have permissions - either running as root, or we have group access to the driver
+	fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
+	if err != nil {
+		if errors.Is(err, fs.ErrPermission) {
+			log.Fatalf("Radeon card detected, but permissions not set up properly.  Either run ollama as root, or add you user account to the render group.")
+			return err
+		} else if errors.Is(err, fs.ErrNotExist) {
+			// expected behavior without a radeon card
+			return nil
+		}
+
+		return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
+	}
+	fd.Close()
+
+	shimMutex.Lock()
+	defer shimMutex.Unlock()
+	if llm != nil {
+		return nil
+	}
+	var libName string
+	switch runtime.GOOS {
+	case "darwin":
+		// shouldn't happen
+		return nil
+	case "linux":
+		libName = "librocm_server.so"
+	case "windows":
+		libName = "rocm_server.dll"
+	default:
+		// shouldn't happen
+		return nil
+	}
+	libPath := C.CString(filepath.Join(workdir, libName))
+	defer C.free(unsafe.Pointer(libPath))
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	var srv C.struct_rocm_llama_server
+	C.rocm_shim_init(libPath, &srv, &resp)
+	if resp.id < 0 {
+		// TODO - consider softening this failure mode to allow fall-back to the CUDA based built-in llm
+		//        and run against CPU
+		return fmt.Errorf("Unable to load AMD GPU library: %s", C.GoString(resp.msg))
+	}
+	llm = &shimExtServer{
+		s:       srv,
+		options: api.DefaultOptions(),
+	}
+	return nil
+}
+
+func extractLib(workDir string) error {
+	files, err := fs.Glob(libEmbed, "llama.cpp/gguf/build/*/lib/*rocm_server*")
+	if err != nil || len(files) == 0 {
+		// this is expected, ollama may be compiled without shim library packed in
+		return RocmShimMissing
+	}
+
+	if len(files) != 1 {
+		return fmt.Errorf("build error - multiple rocm libraries detected: %v", files)
+	}
+
+	srcFile, err := libEmbed.Open(files[0])
+	if err != nil {
+		return fmt.Errorf("read ROCm shim %s: %v", files[0], err)
+	}
+	defer srcFile.Close()
+	if err := os.MkdirAll(workDir, 0o755); err != nil {
+		return fmt.Errorf("create ROCm shim temp dir %s: %v", workDir, err)
+	}
+
+	destFile := filepath.Join(workDir, filepath.Base(files[0]))
+
+	_, err = os.Stat(destFile)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+		if err != nil {
+			return fmt.Errorf("write ROCm shim %s: %v", files[0], err)
+		}
+		defer destFile.Close()
+		if _, err := io.Copy(destFile, srcFile); err != nil {
+			return fmt.Errorf("copy ROCm shim %s: %v", files[0], err)
+		}
+	case err != nil:
+		return fmt.Errorf("stat ROCm shim %s: %v", files[0], err)
+	}
+	NoShim = false
+	return nil
+}

scripts/build_darwin.sh

@@ -9,7 +9,7 @@ mkdir -p dist
 
 for TARGETARCH in arm64 amd64; do
     GOOS=darwin GOARCH=$TARGETARCH go generate ./...
-    GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH
+    CGO_ENABLED=1 GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH
     rm -rf llm/llama.cpp/*/build
 done
 

scripts/build_linux.sh

@@ -7,8 +7,8 @@ export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version
 
 mkdir -p dist
 
-for TARGETARCH in arm64 amd64; do
-    docker buildx build --load --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
+for TARGETARCH in amd64 arm64; do
+    docker buildx build --load --progress=plain --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
     docker create --platform linux/$TARGETARCH --name builder-$TARGETARCH builder:$TARGETARCH
     docker cp builder-$TARGETARCH:/go/src/github.com/jmorganca/ollama/ollama ./dist/ollama-linux-$TARGETARCH
     docker rm builder-$TARGETARCH

scripts/build_remote.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+import subprocess
+import sys
+from urllib.parse import urlparse
+from git import Repo
+# TODO - add argpare and make this more configurable 
+# - force flag becomes optional
+# - build or test as
+
+# Note: remote repo will need this run once:
+# git config --local receive.denyCurrentBranch updateInstead
+repo = Repo(".")
+# GoCmd = "/usr/local/go/bin/go" 
+GoCmd = "go" 
+
+if repo.is_dirty():
+    print("Tree is dirty.  Commit your changes before running this script")
+    sys.exit(1)
+
+if len(sys.argv) != 2:
+    print("Please specify the remote name: " + ', '.join([r.name for r in repo.remotes]))
+    sys.exit(1)
+remote_name = sys.argv[1]
+
+remote = {r.name: r for r in repo.remotes}[remote_name]
+raw_url = list(remote.urls)[0]
+url = urlparse(raw_url)
+# Windows urls don't quite parse properly
+if url.scheme == "" and url.netloc == "":
+    url = urlparse("ssh://" + raw_url)
+print("URL: " + str(url))
+netloc = url.netloc.split(":")[0]
+path = url.path
+branch_name = repo.active_branch.name
+
+print("Force pushing content to remote...")
+# Use with care given the force push
+remote.push(force=True).raise_if_error()
+
+print("Ensuring correct branch checked out on remote via ssh...")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'git', 'checkout', branch_name])
+
+
+# TODO - add some hardening to try to figure out how to set up the path properly
+# subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'env'])
+# TODO - or consider paramiko maybe
+
+print("Performing generate")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'generate', './...'])
+
+print("Building")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
+

scripts/setup_integration_tests.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+# This script sets up integration tests which run the full stack to verify
+# inference locally
+set -e
+set -o pipefail
+
+REPO=$(dirname $0)/../
+export OLLAMA_MODELS=${REPO}/test_data/models
+REGISTRY_SCHEME=https
+REGISTRY=registry.ollama.ai
+TEST_MODEL=library/orca-mini
+TEST_MODEL_TAG=latest
+ACCEPT_HEADER="Accept: application/vnd.docker.distribution.manifest.v2+json"
+
+mkdir -p ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/
+mkdir -p ${OLLAMA_MODELS}/blobs/
+
+echo "Pulling manifest for ${TEST_MODEL}:${TEST_MODEL_TAG}"
+curl -s --header "${ACCEPT_HEADER}" \
+    -o ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} \
+    ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/manifests/${TEST_MODEL_TAG} 
+
+CFG_HASH=$(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".config.digest")
+echo "Pulling config blob ${CFG_HASH}"
+curl -L -C - --header "${ACCEPT_HEADER}" \
+        -o ${OLLAMA_MODELS}/blobs/${CFG_HASH} \
+        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${CFG_HASH}
+
+for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".layers[].digest" ) ; do
+    echo "Pulling blob ${LAYER}"
+    curl -L -C - --header "${ACCEPT_HEADER}" \
+        -o ${OLLAMA_MODELS}/blobs/${LAYER} \
+        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
+done

server/images.go

@@ -418,6 +418,31 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
 					return err
 				}
 
+				// if the model is not in gguf format, pull the base model to try and get it in gguf format
+				if fromConfig.ModelFormat != "gguf" {
+					fn(api.ProgressResponse{Status: "updating base model"})
+					parent, err := GetModel(c.Args)
+					if err != nil {
+						return err
+					}
+					if err := PullModel(ctx, parent.OriginalModel, &RegistryOptions{}, fn); err != nil {
+						log.Printf("error pulling model: %v", err)
+					}
+					// Reset the file pointer to the beginning of the file
+					_, err = fromConfigFile.Seek(0, 0)
+					if err != nil {
+						return fmt.Errorf("update from config after pull: %w", err)
+					}
+					if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
+						return err
+					}
+				}
+
+				// if the model is still not in gguf format, error out
+				if fromConfig.ModelFormat != "gguf" {
+					return fmt.Errorf("%s is not in gguf format, this base model is not compatible with this version of ollama", c.Args)
+				}
+
 				config.SetModelFormat(fromConfig.ModelFormat)
 				config.SetModelFamily(append(fromConfig.ModelFamilies, fromConfig.ModelFamily)...)
 				config.SetModelType(fromConfig.ModelType)
@@ -456,15 +481,21 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
 			defer bin.Close()
 
 			var offset int64
+		CREATE:
 			for {
 				fn(api.ProgressResponse{Status: "creating model layer"})
 
 				bin.Seek(offset, io.SeekStart)
 				ggml, err := llm.DecodeGGML(bin)
-				if errors.Is(err, io.EOF) {
-					break
-				} else if err != nil {
-					return err
+				if err != nil {
+					switch {
+					case errors.Is(err, io.EOF):
+						break CREATE
+					case errors.Is(err, llm.ErrUnsupportedFormat):
+						return fmt.Errorf("model binary specified in FROM field is not a valid gguf format model, %w", err)
+					default:
+						return err
+					}
 				}
 
 				config.SetModelFormat(ggml.Name())

server/images_test.go

@@ -81,7 +81,7 @@ func TestChat(t *testing.T) {
 			Template: tt.template,
 		}
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := m.ChatPrompt(tt.msgs)
+			got, _, err := m.ChatPrompt(tt.msgs)
 			if tt.wantErr != "" {
 				if err == nil {
 					t.Errorf("ChatPrompt() expected error, got nil")

server/llm_test.go

@@ -0,0 +1,121 @@
+package server
+
+import (
+	"context"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/llm"
+)
+
+// TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
+//        package to avoid circular dependencies
+
+// WARNING - these tests will fail on mac if you don't manually copy ggml-metal.metal to this dir (./server)
+//
+// TODO - Fix this ^^
+
+var (
+	req = [2]api.GenerateRequest{
+		{
+			Model:   "orca-mini",
+			Prompt:  "tell me a short story about agi?",
+			Options: map[string]interface{}{},
+		}, {
+			Model:   "orca-mini",
+			Prompt:  "what is the origin of the us thanksgiving holiday?",
+			Options: map[string]interface{}{},
+		},
+	}
+	resp = [2]string{
+		"once upon a time",
+		"united states thanksgiving",
+	}
+)
+
+func TestIntegrationSimpleOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+	defer llmRunner.Close()
+	response := OneShotPromptResponse(t, ctx, req[0], model, llmRunner)
+	assert.Contains(t, strings.ToLower(response), resp[0])
+}
+
+// TODO
+// The server always loads a new runner and closes the old one, which forces serial execution
+// At present this test case fails with concurrency problems.  Eventually we should try to
+// get true concurrency working with n_parallel support in the backend
+func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+
+	t.Skip("concurrent prediction on single runner not currently supported")
+
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	var wg sync.WaitGroup
+	wg.Add(len(req))
+	model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+	defer llmRunner.Close()
+	for i := 0; i < len(req); i++ {
+		go func(i int) {
+			defer wg.Done()
+			response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
+			t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
+			assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
+		}(i)
+	}
+	wg.Wait()
+}
+
+func TestIntegrationConcurrentRunnersOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	var wg sync.WaitGroup
+	wg.Add(len(req))
+
+	t.Logf("Running %d concurrently", len(req))
+	for i := 0; i < len(req); i++ {
+		go func(i int) {
+			defer wg.Done()
+			model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+			defer llmRunner.Close()
+			response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
+			t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
+			assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
+		}(i)
+	}
+	wg.Wait()
+}
+
+// TODO - create a parallel test with 2 different models once we support concurrency

server/llm_utils_test.go

@@ -0,0 +1,76 @@
+package server
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/llm"
+	"github.com/stretchr/testify/require"
+)
+
+func SkipIFNoTestData(t *testing.T) {
+	modelDir := getModelDir()
+	if _, err := os.Stat(modelDir); errors.Is(err, os.ErrNotExist) {
+		t.Skipf("%s does not exist - skipping integration tests", modelDir)
+	}
+}
+
+func getModelDir() string {
+	_, filename, _, _ := runtime.Caller(0)
+	return path.Dir(path.Dir(filename) + "/../test_data/models/.")
+}
+
+func PrepareModelForPrompts(t *testing.T, modelName string, opts api.Options) (*Model, llm.LLM) {
+	modelDir := getModelDir()
+	os.Setenv("OLLAMA_MODELS", modelDir)
+	model, err := GetModel(modelName)
+	require.NoError(t, err, "GetModel ")
+	err = opts.FromMap(model.Options)
+	require.NoError(t, err, "opts from model ")
+	runner, err := llm.New("unused", model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
+	require.NoError(t, err, "llm.New failed")
+	return model, runner
+}
+
+func OneShotPromptResponse(t *testing.T, ctx context.Context, req api.GenerateRequest, model *Model, runner llm.LLM) string {
+	checkpointStart := time.Now()
+	prompt, err := model.Prompt(PromptVars{
+		System: req.System,
+		Prompt: req.Prompt,
+		First:  len(req.Context) == 0,
+	})
+	require.NoError(t, err, "prompt generation failed")
+	success := make(chan bool, 1)
+	response := ""
+	cb := func(r llm.PredictResult) {
+
+		if !r.Done {
+			response += r.Content
+		} else {
+			success <- true
+		}
+	}
+	checkpointLoaded := time.Now()
+	predictReq := llm.PredictOpts{
+		Prompt:           prompt,
+		Format:           req.Format,
+		CheckpointStart:  checkpointStart,
+		CheckpointLoaded: checkpointLoaded,
+	}
+	err = runner.Predict(ctx, predictReq, cb)
+	require.NoError(t, err, "predict call failed")
+
+	select {
+	case <-ctx.Done():
+		t.Errorf("failed to complete before timeout: \n%s", response)
+		return ""
+	case <-success:
+		return response
+	}
+}

server/routes.go

@@ -25,6 +25,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/gpu"
 	"github.com/jmorganca/ollama/llm"
 	"github.com/jmorganca/ollama/parser"
 	"github.com/jmorganca/ollama/version"
@@ -110,7 +111,7 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 			// some older models are not compatible with newer versions of llama.cpp
 			// show a generalized compatibility error until there is a better way to
 			// check for model compatibility
-			if strings.Contains(err.Error(), "failed to load model") {
+			if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
 				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
 			}
 
@@ -122,10 +123,6 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 		loaded.Options = &opts
 	}
 
-	// update options for the loaded llm
-	// TODO(mxyng): this isn't thread safe, but it should be fine for now
-	loaded.runner.SetOptions(opts)
-
 	loaded.expireAt = time.Now().Add(sessionDuration)
 
 	if loaded.expireTimer == nil {
@@ -884,9 +881,12 @@ func Serve(ln net.Listener, allowOrigins []string) error {
 		os.Exit(0)
 	}()
 
-	if runtime.GOOS == "linux" {
+	if err := llm.Init(workDir); err != nil {
+		return fmt.Errorf("unable to initialize llm library %w", err)
+	}
+	if runtime.GOOS == "linux" { // TODO - windows too
 		// check compatibility to log warnings
-		if _, err := llm.CheckVRAM(); err != nil {
+		if _, err := gpu.CheckVRAM(); err != nil {
 			log.Print(err.Error())
 		}
 	}