diff --git a/llm/dyn_ext_server.go b/llm/dyn_ext_server.go index 45b8da12..4e9dbdee 100644 --- a/llm/dyn_ext_server.go +++ b/llm/dyn_ext_server.go @@ -194,6 +194,7 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu request["grammar"] = jsonGrammar } + var whitespace int retryDelay := 100 * time.Microsecond for retries := 0; retries < maxRetries; retries++ { if retries > 0 { @@ -252,6 +253,24 @@ func (llm *dynExtServer) Predict(ctx context.Context, predict PredictOpts, fn fu break out } + // detect if p.Content is entirely whitespace + if predict.Format == "json" && strings.TrimSpace(p.Content) == "" { + whitespace++ + + // if we get 100 consecutive whitespace responses, cancel + if whitespace > 100 { + slog.Debug("cancelling due to excessive whitespace") + C.dyn_llama_server_completion_cancel(llm.s, resp.id, &resp) + if resp.id < 0 { + return extServerResponseToErr(resp) + } + + return nil + } + } else { + whitespace = 0 + } + if p.Content != "" { fn(PredictResult{ Content: p.Content,