From 8f440d579aad22faf191ef2f7e9b38b4f614e070 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 24 May 2024 16:01:37 -0700 Subject: [PATCH 1/2] fix q5_0, q5_1 --- llm/ggml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/ggml.go b/llm/ggml.go index 9b6da425..3127eefc 100644 --- a/llm/ggml.go +++ b/llm/ggml.go @@ -127,7 +127,7 @@ func (t Tensor) blockSize() uint64 { switch t.Kind { case 0, 1, 24, 25, 26, 27, 28, 31: // F32, F16, I8, I16, I32, I64, F64, BF16 return 1 - case 2, 3, 8, 9, 20: // Q4_0, Q4_1, Q8_0, Q8_1, IQ4_NL + case 2, 3, 4, 5, 6, 7, 8, 9, 20: // Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, IQ4_NL return 32 default: // All others return 256 From d51f15257c24e3954d828bde3f25348c7561f440 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 24 May 2024 16:10:43 -0700 Subject: [PATCH 2/2] Update llm/ggml.go Co-authored-by: Bruce MacDonald --- llm/ggml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm/ggml.go b/llm/ggml.go index 3127eefc..878800f3 100644 --- a/llm/ggml.go +++ b/llm/ggml.go @@ -125,7 +125,7 @@ type Tensor struct { func (t Tensor) blockSize() uint64 { switch t.Kind { - case 0, 1, 24, 25, 26, 27, 28, 31: // F32, F16, I8, I16, I32, I64, F64, BF16 + case 0, 1, 24, 25, 26, 27, 28, 30: // F32, F16, I8, I16, I32, I64, F64, BF16 return 1 case 2, 3, 4, 5, 6, 7, 8, 9, 20: // Q4_0, Q4_1, Q5_0, Q5_1, Q8_0, Q8_1, IQ4_NL return 32