72 lines
2.6 KiB
Plaintext
72 lines
2.6 KiB
Plaintext
/**
|
|
* llama.cpp - commit ee459f40f65810a810151b24eba5b8bd174ceffe - do not edit this file
|
|
*
|
|
* MIT License
|
|
*
|
|
* Copyright (c) 2023-2024 The ggml authors
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
* of this software and associated documentation files (the "Software"), to deal
|
|
* in the Software without restriction, including without limitation the rights
|
|
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
* copies of the Software, and to permit persons to whom the Software is
|
|
* furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice shall be included in all
|
|
* copies or substantial portions of the Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
* SOFTWARE.
|
|
*/
|
|
|
|
#include "quantize.cuh"
|
|
|
|
static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int64_t kx, const int64_t kx_padded) {
|
|
const int64_t ix = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
|
|
|
|
if (ix >= kx_padded) {
|
|
return;
|
|
}
|
|
|
|
const int64_t iy = (int64_t)blockDim.y*blockIdx.y + threadIdx.y;
|
|
|
|
const int64_t i_padded = (int64_t)iy*kx_padded + ix;
|
|
|
|
block_q8_1 * y = (block_q8_1 *) vy;
|
|
|
|
const int64_t ib = i_padded / QK8_1; // block index
|
|
const int64_t iqs = i_padded % QK8_1; // quant index
|
|
|
|
const float xi = ix < kx ? x[iy*kx + ix] : 0.0f;
|
|
float amax = fabsf(xi);
|
|
float sum = xi;
|
|
|
|
amax = warp_reduce_max(amax);
|
|
sum = warp_reduce_sum(sum);
|
|
|
|
const float d = amax / 127;
|
|
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
|
|
|
|
y[ib].qs[iqs] = q;
|
|
|
|
if (iqs > 0) {
|
|
return;
|
|
}
|
|
|
|
reinterpret_cast<half&>(y[ib].ds.x) = d;
|
|
reinterpret_cast<half&>(y[ib].ds.y) = sum;
|
|
}
|
|
|
|
void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream) {
|
|
const int64_t block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
|
|
const dim3 num_blocks(block_num_x, ky, 1);
|
|
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
|
|
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);
|
|
}
|
|
|