third-party-mirrors/ollama
1
0
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge d93fbf6a60b95234656b2c6b2f734cf4429a716d into d7eb05b9361febead29a74e71ddffc2ebeff5302

Browse Source
This commit is contained in:
zsmooter 2024-11-14 13:56:25 +08:00 committed by GitHub
commit fd22c9706a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llm/server.go
View File

@ -238,10 +238,10 @@ func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapter
params = append(params, "--flash-attn")
}
// Windows CUDA should not use mmap for best performance
// Windows CUDA/ROCm should not use mmap for best performance
// Linux with a model larger than free space, mmap leads to thrashing
// For CPU loads we want the memory to be allocated, not FS cache
if (runtime.GOOS == "windows" && gpus[0].Library == "cuda" && opts.UseMMap == nil) ||
if (runtime.GOOS == "windows" && (gpus[0].Library == "cuda" || gpus[0].Library == "rocm") && opts.UseMMap == nil) ||
(runtime.GOOS == "linux" && systemFreeMemory < estimate.TotalSize && opts.UseMMap == nil) ||
(gpus[0].Library == "cpu" && opts.UseMMap == nil) ||
(opts.UseMMap != nil && !*opts.UseMMap) {