From dbba73469d98fc45d255dd526dedb201548d823d Mon Sep 17 00:00:00 2001 From: Daniel Hiltgen Date: Sat, 21 Sep 2024 16:54:49 -0700 Subject: [PATCH] runner: Set windows above normal priority (#6905) When running the subprocess as a background service windows may throttle, which can lead to thrashing and very poor token rate. --- llm/llm_windows.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/llm/llm_windows.go b/llm/llm_windows.go index 74a735c2..915355a2 100644 --- a/llm/llm_windows.go +++ b/llm/llm_windows.go @@ -4,7 +4,10 @@ import ( "syscall" ) -const CREATE_DEFAULT_ERROR_MODE = 0x04000000 +const ( + CREATE_DEFAULT_ERROR_MODE = 0x04000000 + ABOVE_NORMAL_PRIORITY_CLASS = 0x00008000 +) var LlamaServerSysProcAttr = &syscall.SysProcAttr{ // Wire up the default error handling logic If for some reason a DLL is @@ -12,5 +15,8 @@ var LlamaServerSysProcAttr = &syscall.SysProcAttr{ // the user can either fix their PATH, or report a bug. Without this // setting, the process exits immediately with a generic exit status but no // way to (easily) figure out what the actual missing DLL was. - CreationFlags: CREATE_DEFAULT_ERROR_MODE, + // + // Setting Above Normal priority class ensures when running as a "background service" + // with "programs" given best priority, we aren't starved of cpu cycles + CreationFlags: CREATE_DEFAULT_ERROR_MODE | ABOVE_NORMAL_PRIORITY_CLASS, }