forked from third-party-mirrors/ollama
replace static build in llm
This commit is contained in:
jmorganca
parent
ec09be97e8
commit
01ccbc07fe
1
.gitignore
vendored
1
.gitignore
vendored
@ -5,7 +5,6 @@
|
|||||||
.swp
|
.swp
|
||||||
dist
|
dist
|
||||||
ollama
|
ollama
|
||||||
ggml-metal.metal
|
|
||||||
.cache
|
.cache
|
||||||
*.exe
|
*.exe
|
||||||
.idea
|
.idea
|
||||||
|
|||||||
1970
llama/ggml-alloc.c
vendored
1970
llama/ggml-alloc.c
vendored
File diff suppressed because it is too large
Load Diff
152
llama/ggml-alloc.h
vendored
152
llama/ggml-alloc.h
vendored
@ -1,76 +1,76 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
|
typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
|
||||||
typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
|
typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
|
||||||
typedef struct ggml_backend * ggml_backend_t;
|
typedef struct ggml_backend * ggml_backend_t;
|
||||||
|
|
||||||
// Tensor allocator
|
// Tensor allocator
|
||||||
struct ggml_tallocr {
|
struct ggml_tallocr {
|
||||||
ggml_backend_buffer_t buffer;
|
ggml_backend_buffer_t buffer;
|
||||||
void * base;
|
void * base;
|
||||||
size_t alignment;
|
size_t alignment;
|
||||||
size_t offset;
|
size_t offset;
|
||||||
};
|
};
|
||||||
|
|
||||||
GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);
|
GGML_API struct ggml_tallocr ggml_tallocr_new(ggml_backend_buffer_t buffer);
|
||||||
GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);
|
GGML_API void ggml_tallocr_alloc(struct ggml_tallocr * talloc, struct ggml_tensor * tensor);
|
||||||
|
|
||||||
// Graph allocator
|
// Graph allocator
|
||||||
/*
|
/*
|
||||||
Example usage:
|
Example usage:
|
||||||
ggml_gallocr_t galloc = ggml_gallocr_new(ggml_bacckend_cpu_buffer_type());
|
ggml_gallocr_t galloc = ggml_gallocr_new(ggml_bacckend_cpu_buffer_type());
|
||||||
|
|
||||||
// optional: create a worst-case graph and reserve the buffers to avoid reallocations
|
// optional: create a worst-case graph and reserve the buffers to avoid reallocations
|
||||||
ggml_gallocr_reserve(galloc, build_graph(max_batch));
|
ggml_gallocr_reserve(galloc, build_graph(max_batch));
|
||||||
|
|
||||||
// allocate the graph
|
// allocate the graph
|
||||||
struct ggml_cgraph * graph = build_graph(batch);
|
struct ggml_cgraph * graph = build_graph(batch);
|
||||||
ggml_gallocr_alloc_graph(galloc, graph);
|
ggml_gallocr_alloc_graph(galloc, graph);
|
||||||
|
|
||||||
printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));
|
printf("compute buffer size: %zu bytes\n", ggml_gallocr_get_buffer_size(galloc, 0));
|
||||||
|
|
||||||
// evaluate the graph
|
// evaluate the graph
|
||||||
ggml_backend_graph_compute(backend, graph);
|
ggml_backend_graph_compute(backend, graph);
|
||||||
*/
|
*/
|
||||||
|
|
||||||
// special tensor flags for use with the graph allocator:
|
// special tensor flags for use with the graph allocator:
|
||||||
// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
|
// ggml_set_input(): all input tensors are allocated at the beginning of the graph in non-overlapping addresses
|
||||||
// ggml_set_output(): output tensors are never freed and never overwritten
|
// ggml_set_output(): output tensors are never freed and never overwritten
|
||||||
|
|
||||||
typedef struct ggml_gallocr * ggml_gallocr_t;
|
typedef struct ggml_gallocr * ggml_gallocr_t;
|
||||||
|
|
||||||
GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);
|
GGML_API ggml_gallocr_t ggml_gallocr_new(ggml_backend_buffer_type_t buft);
|
||||||
GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);
|
GGML_API ggml_gallocr_t ggml_gallocr_new_n(ggml_backend_buffer_type_t * bufts, int n_bufs);
|
||||||
GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);
|
GGML_API void ggml_gallocr_free(ggml_gallocr_t galloc);
|
||||||
|
|
||||||
// pre-allocate buffers from a measure graph - does not allocate or modify the graph
|
// pre-allocate buffers from a measure graph - does not allocate or modify the graph
|
||||||
// call with a worst-case graph to avoid buffer reallocations
|
// call with a worst-case graph to avoid buffer reallocations
|
||||||
// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
|
// not strictly required for single buffer usage: ggml_gallocr_alloc_graph will reallocate the buffers automatically if needed
|
||||||
// returns false if the buffer allocation failed
|
// returns false if the buffer allocation failed
|
||||||
GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
|
GGML_API bool ggml_gallocr_reserve(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
|
||||||
GGML_API bool ggml_gallocr_reserve_n(
|
GGML_API bool ggml_gallocr_reserve_n(
|
||||||
ggml_gallocr_t galloc,
|
ggml_gallocr_t galloc,
|
||||||
struct ggml_cgraph * graph,
|
struct ggml_cgraph * graph,
|
||||||
const int * node_buffer_ids,
|
const int * node_buffer_ids,
|
||||||
const int * leaf_buffer_ids);
|
const int * leaf_buffer_ids);
|
||||||
|
|
||||||
// automatic reallocation if the topology changes when using a single buffer
|
// automatic reallocation if the topology changes when using a single buffer
|
||||||
// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)
|
// returns false if using multiple buffers and a re-allocation is needed (call ggml_gallocr_reserve_n first to set the node buffers)
|
||||||
GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
|
GGML_API bool ggml_gallocr_alloc_graph(ggml_gallocr_t galloc, struct ggml_cgraph * graph);
|
||||||
|
|
||||||
GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);
|
GGML_API size_t ggml_gallocr_get_buffer_size(ggml_gallocr_t galloc, int buffer_id);
|
||||||
|
|
||||||
// Utils
|
// Utils
|
||||||
// Create a buffer and allocate all the tensors in a ggml_context
|
// Create a buffer and allocate all the tensors in a ggml_context
|
||||||
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
|
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors_from_buft(struct ggml_context * ctx, ggml_backend_buffer_type_t buft);
|
||||||
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);
|
GGML_API struct ggml_backend_buffer * ggml_backend_alloc_ctx_tensors(struct ggml_context * ctx, ggml_backend_t backend);
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
282
llama/ggml-backend-impl.h
vendored
282
llama/ggml-backend-impl.h
vendored
@ -1,141 +1,141 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
// ggml-backend internal header
|
// ggml-backend internal header
|
||||||
|
|
||||||
#include "ggml-backend.h"
|
#include "ggml-backend.h"
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend buffer
|
// Backend buffer
|
||||||
//
|
//
|
||||||
|
|
||||||
// buffer type
|
// buffer type
|
||||||
typedef void * ggml_backend_buffer_type_context_t;
|
typedef void * ggml_backend_buffer_type_context_t;
|
||||||
|
|
||||||
struct ggml_backend_buffer_type_i {
|
struct ggml_backend_buffer_type_i {
|
||||||
const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft);
|
const char * (*GGML_CALL get_name) (ggml_backend_buffer_type_t buft);
|
||||||
ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
|
ggml_backend_buffer_t (*GGML_CALL alloc_buffer) (ggml_backend_buffer_type_t buft, size_t size);
|
||||||
size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment
|
size_t (*GGML_CALL get_alignment) (ggml_backend_buffer_type_t buft); // tensor alignment
|
||||||
size_t (*GGML_CALL get_max_size) (ggml_backend_buffer_type_t buft); // allocation max size
|
size_t (*GGML_CALL get_max_size) (ggml_backend_buffer_type_t buft); // allocation max size
|
||||||
size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding
|
size_t (*GGML_CALL get_alloc_size) (ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor); // data size needed to allocate the tensor, including padding
|
||||||
bool (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend
|
bool (*GGML_CALL supports_backend)(ggml_backend_buffer_type_t buft, ggml_backend_t backend); // check if the buffer type is usable by the backend
|
||||||
// check if tensor data is in host memory
|
// check if tensor data is in host memory
|
||||||
// should be equivalent to supports_backend(buft, ggml_backend_cpu_init())
|
// should be equivalent to supports_backend(buft, ggml_backend_cpu_init())
|
||||||
bool (*GGML_CALL is_host) (ggml_backend_buffer_type_t buft);
|
bool (*GGML_CALL is_host) (ggml_backend_buffer_type_t buft);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ggml_backend_buffer_type {
|
struct ggml_backend_buffer_type {
|
||||||
struct ggml_backend_buffer_type_i iface;
|
struct ggml_backend_buffer_type_i iface;
|
||||||
ggml_backend_buffer_type_context_t context;
|
ggml_backend_buffer_type_context_t context;
|
||||||
};
|
};
|
||||||
|
|
||||||
// buffer
|
// buffer
|
||||||
typedef void * ggml_backend_buffer_context_t;
|
typedef void * ggml_backend_buffer_context_t;
|
||||||
|
|
||||||
struct ggml_backend_buffer_i {
|
struct ggml_backend_buffer_i {
|
||||||
const char * (*GGML_CALL get_name) (ggml_backend_buffer_t buffer);
|
const char * (*GGML_CALL get_name) (ggml_backend_buffer_t buffer);
|
||||||
void (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);
|
void (*GGML_CALL free_buffer)(ggml_backend_buffer_t buffer);
|
||||||
void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer);
|
void * (*GGML_CALL get_base) (ggml_backend_buffer_t buffer);
|
||||||
void (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
void (*GGML_CALL init_tensor)(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
||||||
void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
void (*GGML_CALL set_tensor) (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||||
void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
void (*GGML_CALL get_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||||
bool (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
|
bool (*GGML_CALL cpy_tensor) (ggml_backend_buffer_t buffer, const struct ggml_tensor * src, struct ggml_tensor * dst); // dst is in the buffer, src may be in any buffer
|
||||||
void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value);
|
void (*GGML_CALL clear) (ggml_backend_buffer_t buffer, uint8_t value);
|
||||||
void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
|
void (*GGML_CALL reset) (ggml_backend_buffer_t buffer); // reset any internal state due to tensor initialization, such as tensor extras
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ggml_backend_buffer {
|
struct ggml_backend_buffer {
|
||||||
struct ggml_backend_buffer_i iface;
|
struct ggml_backend_buffer_i iface;
|
||||||
ggml_backend_buffer_type_t buft;
|
ggml_backend_buffer_type_t buft;
|
||||||
ggml_backend_buffer_context_t context;
|
ggml_backend_buffer_context_t context;
|
||||||
size_t size;
|
size_t size;
|
||||||
enum ggml_backend_buffer_usage usage;
|
enum ggml_backend_buffer_usage usage;
|
||||||
};
|
};
|
||||||
|
|
||||||
GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
|
GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
|
||||||
ggml_backend_buffer_type_t buft,
|
ggml_backend_buffer_type_t buft,
|
||||||
struct ggml_backend_buffer_i iface,
|
struct ggml_backend_buffer_i iface,
|
||||||
ggml_backend_buffer_context_t context,
|
ggml_backend_buffer_context_t context,
|
||||||
size_t size);
|
size_t size);
|
||||||
|
|
||||||
// do not use directly, use ggml_backend_tensor_copy instead
|
// do not use directly, use ggml_backend_tensor_copy instead
|
||||||
bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
|
bool ggml_backend_buffer_copy_tensor(const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||||
|
|
||||||
// buffer that contains a collection of buffers
|
// buffer that contains a collection of buffers
|
||||||
GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
|
GGML_CALL ggml_backend_buffer_t ggml_backend_multi_buffer_alloc_buffer(ggml_backend_buffer_t * buffers, size_t n_buffers);
|
||||||
GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
|
GGML_CALL bool ggml_backend_buffer_is_multi_buffer(ggml_backend_buffer_t buffer);
|
||||||
GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
|
GGML_CALL void ggml_backend_multi_buffer_set_usage(ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend
|
// Backend
|
||||||
//
|
//
|
||||||
|
|
||||||
typedef void * ggml_backend_context_t;
|
typedef void * ggml_backend_context_t;
|
||||||
|
|
||||||
struct ggml_backend_i {
|
struct ggml_backend_i {
|
||||||
const char * (*GGML_CALL get_name)(ggml_backend_t backend);
|
const char * (*GGML_CALL get_name)(ggml_backend_t backend);
|
||||||
|
|
||||||
void (*GGML_CALL free)(ggml_backend_t backend);
|
void (*GGML_CALL free)(ggml_backend_t backend);
|
||||||
|
|
||||||
// buffer allocation
|
// buffer allocation
|
||||||
ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);
|
ggml_backend_buffer_type_t (*GGML_CALL get_default_buffer_type)(ggml_backend_t backend);
|
||||||
|
|
||||||
// (optional) asynchronous tensor data access
|
// (optional) asynchronous tensor data access
|
||||||
void (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
void (*GGML_CALL set_tensor_async)(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||||
void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
void (*GGML_CALL get_tensor_async)(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||||
bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
|
bool (*GGML_CALL cpy_tensor_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||||
|
|
||||||
// (optional) complete all pending operations
|
// (optional) complete all pending operations
|
||||||
void (*GGML_CALL synchronize)(ggml_backend_t backend);
|
void (*GGML_CALL synchronize)(ggml_backend_t backend);
|
||||||
|
|
||||||
// compute graph with a plan (not used currently)
|
// compute graph with a plan (not used currently)
|
||||||
ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
|
ggml_backend_graph_plan_t (*GGML_CALL graph_plan_create) (ggml_backend_t backend, const struct ggml_cgraph * cgraph);
|
||||||
void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
void (*GGML_CALL graph_plan_free) (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
||||||
|
|
||||||
// compute graph with a plan
|
// compute graph with a plan
|
||||||
enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
enum ggml_status (*GGML_CALL graph_plan_compute)(ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
||||||
// compute graph without a plan (async)
|
// compute graph without a plan (async)
|
||||||
enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
enum ggml_status (*GGML_CALL graph_compute) (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
|
|
||||||
// check if the backend supports an operation
|
// check if the backend supports an operation
|
||||||
bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);
|
bool (*GGML_CALL supports_op)(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||||
|
|
||||||
// check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
|
// check if the backend wants to run an operation, even if the weights are allocated in a CPU buffer
|
||||||
// these should be expensive operations with large batch sizes that may benefit from running on this backend
|
// these should be expensive operations with large batch sizes that may benefit from running on this backend
|
||||||
// even if the weight has to be copied from the CPU temporarily
|
// even if the weight has to be copied from the CPU temporarily
|
||||||
bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);
|
bool (*GGML_CALL offload_op)(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||||
|
|
||||||
// (optional) event synchronization
|
// (optional) event synchronization
|
||||||
ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend);
|
ggml_backend_event_t (*GGML_CALL event_new) (ggml_backend_t backend);
|
||||||
void (*GGML_CALL event_free) (ggml_backend_event_t event);
|
void (*GGML_CALL event_free) (ggml_backend_event_t event);
|
||||||
void (*GGML_CALL event_record) (ggml_backend_event_t event);
|
void (*GGML_CALL event_record) (ggml_backend_event_t event);
|
||||||
void (*GGML_CALL event_wait) (ggml_backend_t backend, ggml_backend_event_t event);
|
void (*GGML_CALL event_wait) (ggml_backend_t backend, ggml_backend_event_t event);
|
||||||
void (*GGML_CALL event_synchronize) (ggml_backend_event_t event);
|
void (*GGML_CALL event_synchronize) (ggml_backend_event_t event);
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ggml_backend {
|
struct ggml_backend {
|
||||||
ggml_guid_t guid;
|
ggml_guid_t guid;
|
||||||
|
|
||||||
struct ggml_backend_i iface;
|
struct ggml_backend_i iface;
|
||||||
ggml_backend_context_t context;
|
ggml_backend_context_t context;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ggml_backend_event {
|
struct ggml_backend_event {
|
||||||
ggml_backend_t backend;
|
ggml_backend_t backend;
|
||||||
void * context;
|
void * context;
|
||||||
};
|
};
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend registry
|
// Backend registry
|
||||||
//
|
//
|
||||||
|
|
||||||
typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);
|
typedef ggml_backend_t (*GGML_CALL ggml_backend_init_fn)(const char * params, void * user_data);
|
||||||
|
|
||||||
GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);
|
GGML_CALL void ggml_backend_register(const char * name, ggml_backend_init_fn init_fn, ggml_backend_buffer_type_t default_buffer_type, void * user_data);
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
11
llama/ggml-backend.c
vendored
11
llama/ggml-backend.c
vendored
@ -56,6 +56,7 @@ bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// backend buffer
|
// backend buffer
|
||||||
|
|
||||||
GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
|
GGML_CALL ggml_backend_buffer_t ggml_backend_buffer_init(
|
||||||
ggml_backend_buffer_type_t buft,
|
ggml_backend_buffer_type_t buft,
|
||||||
struct ggml_backend_buffer_i iface,
|
struct ggml_backend_buffer_i iface,
|
||||||
@ -78,10 +79,6 @@ const char * ggml_backend_buffer_name(ggml_backend_buffer_t buffer) {
|
|||||||
return buffer->iface.get_name(buffer);
|
return buffer->iface.get_name(buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
#define ggml_assert_aligned(ptr) \
|
|
||||||
GGML_ASSERT(((uintptr_t) (ptr))%GGML_MEM_ALIGN == 0)
|
|
||||||
|
|
||||||
|
|
||||||
void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
|
void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
|
||||||
if (buffer == NULL) {
|
if (buffer == NULL) {
|
||||||
return;
|
return;
|
||||||
@ -90,9 +87,9 @@ void ggml_backend_buffer_free(ggml_backend_buffer_t buffer) {
|
|||||||
if (buffer->iface.free_buffer != NULL) {
|
if (buffer->iface.free_buffer != NULL) {
|
||||||
buffer->iface.free_buffer(buffer);
|
buffer->iface.free_buffer(buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: this needs to be freed in cuda and hipblas backends because
|
// TODO: this needs to be freed in cuda and hipblas backends because
|
||||||
// the cuda backend implementation compiled with msvc
|
// the cuda backend implementation compiled with msvc
|
||||||
#if !defined(GGML_USE_CUDA) && !defined(GGML_USE_HIPBLAS)
|
#if !defined(GGML_USE_CUDA) && !defined(GGML_USE_HIPBLAS)
|
||||||
free(buffer);
|
free(buffer);
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
466
llama/ggml-backend.h
vendored
466
llama/ggml-backend.h
vendored
@ -1,233 +1,233 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
#include "ggml-alloc.h"
|
#include "ggml-alloc.h"
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
|
typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
|
||||||
typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
|
typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
|
||||||
typedef struct ggml_backend_event * ggml_backend_event_t;
|
typedef struct ggml_backend_event * ggml_backend_event_t;
|
||||||
typedef struct ggml_backend * ggml_backend_t;
|
typedef struct ggml_backend * ggml_backend_t;
|
||||||
typedef void * ggml_backend_graph_plan_t;
|
typedef void * ggml_backend_graph_plan_t;
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend buffer
|
// Backend buffer
|
||||||
//
|
//
|
||||||
|
|
||||||
// buffer type
|
// buffer type
|
||||||
GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
|
GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
|
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
|
||||||
GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
|
GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
|
||||||
GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
|
GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
|
||||||
GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);
|
GGML_API GGML_CALL size_t ggml_backend_buft_get_alloc_size (ggml_backend_buffer_type_t buft, struct ggml_tensor * tensor);
|
||||||
GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);
|
GGML_API bool ggml_backend_buft_supports_backend(ggml_backend_buffer_type_t buft, ggml_backend_t backend);
|
||||||
GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
|
GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
|
||||||
|
|
||||||
// buffer
|
// buffer
|
||||||
enum ggml_backend_buffer_usage {
|
enum ggml_backend_buffer_usage {
|
||||||
GGML_BACKEND_BUFFER_USAGE_ANY = 0,
|
GGML_BACKEND_BUFFER_USAGE_ANY = 0,
|
||||||
GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
|
GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
|
||||||
};
|
};
|
||||||
|
|
||||||
GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
|
GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
|
||||||
GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
|
GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
|
||||||
GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
|
GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
|
||||||
GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
|
GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
|
||||||
GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
GGML_API GGML_CALL void ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
||||||
GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
|
GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
|
||||||
GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
|
GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
|
||||||
GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
||||||
GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
|
GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
|
||||||
GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
|
GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
|
||||||
GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
|
GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
|
||||||
GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
|
GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
|
||||||
GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
|
GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend
|
// Backend
|
||||||
//
|
//
|
||||||
|
|
||||||
GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
|
GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
|
||||||
GGML_API const char * ggml_backend_name(ggml_backend_t backend);
|
GGML_API const char * ggml_backend_name(ggml_backend_t backend);
|
||||||
GGML_API void ggml_backend_free(ggml_backend_t backend);
|
GGML_API void ggml_backend_free(ggml_backend_t backend);
|
||||||
|
|
||||||
GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
|
GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
|
||||||
GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
|
GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
|
||||||
GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
|
GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
|
||||||
GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
|
GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
|
||||||
|
|
||||||
GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||||
GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||||
|
|
||||||
GGML_API GGML_CALL void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
GGML_API GGML_CALL void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
|
||||||
GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
GGML_API GGML_CALL void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
|
||||||
|
|
||||||
GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
|
GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
|
||||||
|
|
||||||
GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
||||||
|
|
||||||
GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
|
||||||
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
|
||||||
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||||
GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
|
||||||
|
|
||||||
// tensor copy between different backends
|
// tensor copy between different backends
|
||||||
GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
|
GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||||
|
|
||||||
// asynchronous copy
|
// asynchronous copy
|
||||||
// the copy is performed after all the currently queued operations in backend_src
|
// the copy is performed after all the currently queued operations in backend_src
|
||||||
// backend_dst will wait for the copy to complete before performing other operations
|
// backend_dst will wait for the copy to complete before performing other operations
|
||||||
// automatic fallback to sync copy if async is not supported
|
// automatic fallback to sync copy if async is not supported
|
||||||
GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
|
GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
|
||||||
|
|
||||||
// events
|
// events
|
||||||
GGML_API ggml_backend_event_t ggml_backend_event_new (ggml_backend_t backend);
|
GGML_API ggml_backend_event_t ggml_backend_event_new (ggml_backend_t backend);
|
||||||
GGML_API void ggml_backend_event_free (ggml_backend_event_t event);
|
GGML_API void ggml_backend_event_free (ggml_backend_event_t event);
|
||||||
GGML_API void ggml_backend_event_record (ggml_backend_event_t event);
|
GGML_API void ggml_backend_event_record (ggml_backend_event_t event);
|
||||||
GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
|
GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
|
||||||
GGML_API void ggml_backend_event_wait (ggml_backend_t backend, ggml_backend_event_t event); // wait async on event
|
GGML_API void ggml_backend_event_wait (ggml_backend_t backend, ggml_backend_event_t event); // wait async on event
|
||||||
|
|
||||||
//
|
//
|
||||||
// CPU backend
|
// CPU backend
|
||||||
//
|
//
|
||||||
|
|
||||||
GGML_API ggml_backend_t ggml_backend_cpu_init(void);
|
GGML_API ggml_backend_t ggml_backend_cpu_init(void);
|
||||||
|
|
||||||
GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);
|
GGML_API GGML_CALL bool ggml_backend_is_cpu (ggml_backend_t backend);
|
||||||
GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
|
GGML_API void ggml_backend_cpu_set_n_threads (ggml_backend_t backend_cpu, int n_threads);
|
||||||
GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
|
GGML_API void ggml_backend_cpu_set_abort_callback(ggml_backend_t backend_cpu, ggml_abort_callback abort_callback, void * abort_callback_data);
|
||||||
|
|
||||||
// Create a backend buffer from an existing pointer
|
// Create a backend buffer from an existing pointer
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
|
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
|
||||||
|
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
|
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
|
||||||
|
|
||||||
#ifdef GGML_USE_CPU_HBM
|
#ifdef GGML_USE_CPU_HBM
|
||||||
GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);
|
GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_hbm_buffer_type(void);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend registry
|
// Backend registry
|
||||||
//
|
//
|
||||||
|
|
||||||
// The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
|
// The backend registry is a registry of all the available backends, and allows initializing backends in a generic way
|
||||||
|
|
||||||
GGML_API size_t ggml_backend_reg_get_count(void);
|
GGML_API size_t ggml_backend_reg_get_count(void);
|
||||||
GGML_API size_t ggml_backend_reg_find_by_name(const char * name);
|
GGML_API size_t ggml_backend_reg_find_by_name(const char * name);
|
||||||
GGML_API ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is name[:params]
|
GGML_API ggml_backend_t ggml_backend_reg_init_backend_from_str(const char * backend_str); // str is name[:params]
|
||||||
GGML_API const char * ggml_backend_reg_get_name(size_t i);
|
GGML_API const char * ggml_backend_reg_get_name(size_t i);
|
||||||
GGML_API ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
|
GGML_API ggml_backend_t ggml_backend_reg_init_backend(size_t i, const char * params); // params is backend-specific
|
||||||
GGML_API ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i);
|
GGML_API ggml_backend_buffer_type_t ggml_backend_reg_get_default_buffer_type(size_t i);
|
||||||
GGML_API ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size);
|
GGML_API ggml_backend_buffer_t ggml_backend_reg_alloc_buffer(size_t i, size_t size);
|
||||||
|
|
||||||
//
|
//
|
||||||
// Backend scheduler
|
// Backend scheduler
|
||||||
//
|
//
|
||||||
|
|
||||||
// The backend scheduler allows for multiple backends to be used together
|
// The backend scheduler allows for multiple backends to be used together
|
||||||
// Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
|
// Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
|
||||||
// The backends are selected based on:
|
// The backends are selected based on:
|
||||||
// - the backend that supports the operation
|
// - the backend that supports the operation
|
||||||
// - the location of the pre-allocated tensors (e.g. the weights)
|
// - the location of the pre-allocated tensors (e.g. the weights)
|
||||||
/*
|
/*
|
||||||
Example usage:
|
Example usage:
|
||||||
|
|
||||||
// operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
|
// operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
|
||||||
// preferrably to run on the same backend as the buffer
|
// preferrably to run on the same backend as the buffer
|
||||||
ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
|
||||||
|
|
||||||
sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);
|
sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false);
|
||||||
|
|
||||||
// initialize buffers from a max size graph (optional)
|
// initialize buffers from a max size graph (optional)
|
||||||
reserve_graph = build_graph(sched, max_batch_size);
|
reserve_graph = build_graph(sched, max_batch_size);
|
||||||
|
|
||||||
// manually assign nodes to a backend (optional, should not be needed in most cases)
|
// manually assign nodes to a backend (optional, should not be needed in most cases)
|
||||||
struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
|
struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
|
||||||
ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
|
ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
|
||||||
|
|
||||||
ggml_backend_sched_reserve(sched, reserve_graph);
|
ggml_backend_sched_reserve(sched, reserve_graph);
|
||||||
|
|
||||||
// compute
|
// compute
|
||||||
graph = build_graph(sched);
|
graph = build_graph(sched);
|
||||||
ggml_backend_sched_graph_compute(sched, graph);
|
ggml_backend_sched_graph_compute(sched, graph);
|
||||||
|
|
||||||
// if there are graph inputs:
|
// if there are graph inputs:
|
||||||
ggml_backend_sched_reset(sched);
|
ggml_backend_sched_reset(sched);
|
||||||
ggml_backend_sched_alloc_graph(sched, graph);
|
ggml_backend_sched_alloc_graph(sched, graph);
|
||||||
ggml_backend_tensor_set(input_tensor, ...);
|
ggml_backend_tensor_set(input_tensor, ...);
|
||||||
ggml_backend_sched_graph_compute(sched, graph);
|
ggml_backend_sched_graph_compute(sched, graph);
|
||||||
}
|
}
|
||||||
*/
|
*/
|
||||||
|
|
||||||
struct ggml_backend_sched;
|
struct ggml_backend_sched;
|
||||||
typedef struct ggml_backend_sched * ggml_backend_sched_t;
|
typedef struct ggml_backend_sched * ggml_backend_sched_t;
|
||||||
|
|
||||||
// when ask == true, the scheduler wants to know if the user wants to observe this node
|
// when ask == true, the scheduler wants to know if the user wants to observe this node
|
||||||
// this allows the scheduler to batch nodes together in order to evaluate them in a single call
|
// this allows the scheduler to batch nodes together in order to evaluate them in a single call
|
||||||
//
|
//
|
||||||
// when ask == false, the scheduler is passing the node tensor to the user for observation
|
// when ask == false, the scheduler is passing the node tensor to the user for observation
|
||||||
// if the user returns false, the scheduler will cancel the graph compute
|
// if the user returns false, the scheduler will cancel the graph compute
|
||||||
//
|
//
|
||||||
typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
|
typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
|
||||||
|
|
||||||
// Initialize a backend scheduler
|
// Initialize a backend scheduler
|
||||||
GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
|
GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel);
|
||||||
GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
|
GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
|
||||||
|
|
||||||
// Initialize backend buffers from a measure graph
|
// Initialize backend buffers from a measure graph
|
||||||
GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);
|
GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph);
|
||||||
|
|
||||||
// Get the number of splits of the last graph
|
// Get the number of splits of the last graph
|
||||||
GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
|
GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
|
||||||
GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
|
GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
|
||||||
|
|
||||||
GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
|
GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
|
||||||
|
|
||||||
GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
|
GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
|
||||||
GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
|
GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
|
||||||
|
|
||||||
// Allocate and compute graph on the backend scheduler
|
// Allocate and compute graph on the backend scheduler
|
||||||
GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
||||||
GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
||||||
GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
|
||||||
GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
|
GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
|
||||||
|
|
||||||
// Reset all assignments and allocators - must be called before changing the node backends
|
// Reset all assignments and allocators - must be called before changing the node backends
|
||||||
GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
|
GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
|
||||||
|
|
||||||
// Set a callback to be called for each resulting node during graph compute
|
// Set a callback to be called for each resulting node during graph compute
|
||||||
GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
|
GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
|
||||||
|
|
||||||
//
|
//
|
||||||
// Utils
|
// Utils
|
||||||
//
|
//
|
||||||
|
|
||||||
struct ggml_backend_graph_copy {
|
struct ggml_backend_graph_copy {
|
||||||
ggml_backend_buffer_t buffer;
|
ggml_backend_buffer_t buffer;
|
||||||
struct ggml_context * ctx_allocated;
|
struct ggml_context * ctx_allocated;
|
||||||
struct ggml_context * ctx_unallocated;
|
struct ggml_context * ctx_unallocated;
|
||||||
struct ggml_cgraph * graph;
|
struct ggml_cgraph * graph;
|
||||||
};
|
};
|
||||||
|
|
||||||
// Copy a graph to a different backend
|
// Copy a graph to a different backend
|
||||||
GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
|
GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
|
||||||
GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
|
GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
|
||||||
|
|
||||||
typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
|
typedef bool (*GGML_CALL ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
|
||||||
|
|
||||||
// Compare the output of two backends
|
// Compare the output of two backends
|
||||||
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
|
GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data);
|
||||||
|
|
||||||
// Tensor initialization
|
// Tensor initialization
|
||||||
GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
|
GGML_API void ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
|
||||||
GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
GGML_API void ggml_backend_view_init(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
|
||||||
|
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
3706
llama/ggml-common.h
vendored
3706
llama/ggml-common.h
vendored
File diff suppressed because it is too large
Load Diff
5
llama/ggml-cuda.cu
vendored
5
llama/ggml-cuda.cu
vendored
@ -715,9 +715,6 @@ static bool ggml_backend_buffer_is_cuda_split(ggml_backend_buffer_t buffer) {
|
|||||||
GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
GGML_CALL static void ggml_backend_cuda_split_buffer_free_buffer(ggml_backend_buffer_t buffer) {
|
||||||
ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
|
ggml_backend_cuda_split_buffer_context * ctx = (ggml_backend_cuda_split_buffer_context *)buffer->context;
|
||||||
delete ctx;
|
delete ctx;
|
||||||
|
|
||||||
// HACK: this needs to be freed in msvc
|
|
||||||
free(buffer);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {
|
GGML_CALL static void * ggml_backend_cuda_split_buffer_get_base(ggml_backend_buffer_t buffer) {
|
||||||
@ -3031,7 +3028,7 @@ GGML_CALL static ggml_backend_t ggml_backend_reg_cuda_init(const char * params,
|
|||||||
GGML_UNUSED(params);
|
GGML_UNUSED(params);
|
||||||
}
|
}
|
||||||
|
|
||||||
// extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();
|
extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();
|
||||||
|
|
||||||
GGML_CALL int ggml_backend_cuda_reg_devices() {
|
GGML_CALL int ggml_backend_cuda_reg_devices() {
|
||||||
int device_count = ggml_backend_cuda_get_device_count();
|
int device_count = ggml_backend_cuda_get_device_count();
|
||||||
|
|||||||
88
llama/ggml-cuda.h
vendored
88
llama/ggml-cuda.h
vendored
@ -1,45 +1,43 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
#include "ggml-backend.h"
|
#include "ggml-backend.h"
|
||||||
|
|
||||||
#ifdef GGML_USE_HIPBLAS
|
#ifdef GGML_USE_HIPBLAS
|
||||||
#define GGML_CUDA_NAME "ROCm"
|
#define GGML_CUDA_NAME "ROCm"
|
||||||
#define GGML_CUBLAS_NAME "hipBLAS"
|
#define GGML_CUBLAS_NAME "hipBLAS"
|
||||||
#else
|
#else
|
||||||
#define GGML_CUDA_NAME "CUDA"
|
#define GGML_CUDA_NAME "CUDA"
|
||||||
#define GGML_CUBLAS_NAME "cuBLAS"
|
#define GGML_CUBLAS_NAME "cuBLAS"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#define GGML_CUDA_MAX_DEVICES 16
|
#define GGML_CUDA_MAX_DEVICES 16
|
||||||
|
|
||||||
// backend API
|
// backend API
|
||||||
GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);
|
GGML_API GGML_CALL ggml_backend_t ggml_backend_cuda_init(int device);
|
||||||
|
|
||||||
GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);
|
GGML_API GGML_CALL bool ggml_backend_is_cuda(ggml_backend_t backend);
|
||||||
|
|
||||||
// device buffer
|
// device buffer
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
|
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device);
|
||||||
|
|
||||||
// split tensor buffer that splits matrices by rows across multiple devices
|
// split tensor buffer that splits matrices by rows across multiple devices
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);
|
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(const float * tensor_split);
|
||||||
|
|
||||||
// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
|
// pinned host buffer for use with the CPU backend for faster copies between CPU and GPU
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
|
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);
|
||||||
|
|
||||||
GGML_API GGML_CALL int ggml_backend_cuda_reg_devices();
|
GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);
|
||||||
|
GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
|
||||||
GGML_API GGML_CALL int ggml_backend_cuda_get_device_count(void);
|
GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
|
||||||
GGML_API GGML_CALL void ggml_backend_cuda_get_device_description(int device, char * description, size_t description_size);
|
|
||||||
GGML_API GGML_CALL void ggml_backend_cuda_get_device_memory(int device, size_t * free, size_t * total);
|
GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
|
||||||
|
GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);
|
||||||
GGML_API GGML_CALL bool ggml_backend_cuda_register_host_buffer(void * buffer, size_t size);
|
|
||||||
GGML_API GGML_CALL void ggml_backend_cuda_unregister_host_buffer(void * buffer);
|
#ifdef __cplusplus
|
||||||
|
}
|
||||||
#ifdef __cplusplus
|
#endif
|
||||||
}
|
|
||||||
#endif
|
|
||||||
|
|||||||
94
llama/ggml-cuda/acc.cu
vendored
94
llama/ggml-cuda/acc.cu
vendored
@ -1,47 +1,47 @@
|
|||||||
#include "acc.cuh"
|
#include "acc.cuh"
|
||||||
|
|
||||||
static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,
|
static __global__ void acc_f32(const float * x, const float * y, float * dst, const int ne,
|
||||||
const int ne10, const int ne11, const int ne12,
|
const int ne10, const int ne11, const int ne12,
|
||||||
const int nb1, const int nb2, int offset) {
|
const int nb1, const int nb2, int offset) {
|
||||||
const int i = blockDim.x * blockIdx.x + threadIdx.x;
|
const int i = blockDim.x * blockIdx.x + threadIdx.x;
|
||||||
if (i >= ne) {
|
if (i >= ne) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
int src1_idx = i - offset;
|
int src1_idx = i - offset;
|
||||||
int oz = src1_idx / nb2;
|
int oz = src1_idx / nb2;
|
||||||
int oy = (src1_idx - (oz * nb2)) / nb1;
|
int oy = (src1_idx - (oz * nb2)) / nb1;
|
||||||
int ox = src1_idx % nb1;
|
int ox = src1_idx % nb1;
|
||||||
if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {
|
if (src1_idx >= 0 && ox < ne10 && oy < ne11 && oz < ne12) {
|
||||||
dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];
|
dst[i] = x[i] + y[ox + oy * ne10 + oz * ne10 * ne11];
|
||||||
} else {
|
} else {
|
||||||
dst[i] = x[i];
|
dst[i] = x[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,
|
static void acc_f32_cuda(const float * x, const float * y, float * dst, const int n_elements,
|
||||||
const int ne10, const int ne11, const int ne12,
|
const int ne10, const int ne11, const int ne12,
|
||||||
const int nb1, const int nb2, const int offset, cudaStream_t stream) {
|
const int nb1, const int nb2, const int offset, cudaStream_t stream) {
|
||||||
int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;
|
int num_blocks = (n_elements + CUDA_ACC_BLOCK_SIZE - 1) / CUDA_ACC_BLOCK_SIZE;
|
||||||
acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);
|
acc_f32<<<num_blocks, CUDA_ACC_BLOCK_SIZE, 0, stream>>>(x, y, dst, n_elements, ne10, ne11, ne12, nb1, nb2, offset);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const ggml_tensor * src1 = dst->src[1];
|
const ggml_tensor * src1 = dst->src[1];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
const float * src1_d = (const float *)src1->data;
|
const float * src1_d = (const float *)src1->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported
|
GGML_ASSERT(dst->ne[3] == 1); // just 3D tensors supported
|
||||||
|
|
||||||
int nb1 = dst->op_params[0] / 4; // 4 bytes of float32
|
int nb1 = dst->op_params[0] / 4; // 4 bytes of float32
|
||||||
int nb2 = dst->op_params[1] / 4; // 4 bytes of float32
|
int nb2 = dst->op_params[1] / 4; // 4 bytes of float32
|
||||||
// int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused
|
// int nb3 = dst->op_params[2] / 4; // 4 bytes of float32 - unused
|
||||||
int offset = dst->op_params[3] / 4; // offset in bytes
|
int offset = dst->op_params[3] / 4; // offset in bytes
|
||||||
|
|
||||||
acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, stream);
|
acc_f32_cuda(src0_d, src1_d, dst_d, ggml_nelements(dst), src1->ne[0], src1->ne[1], src1->ne[2], nb1, nb2, offset, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/acc.cuh
vendored
10
llama/ggml-cuda/acc.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_ACC_BLOCK_SIZE 256
|
#define CUDA_ACC_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_acc(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
68
llama/ggml-cuda/arange.cu
vendored
68
llama/ggml-cuda/arange.cu
vendored
@ -1,34 +1,34 @@
|
|||||||
#include "arange.cuh"
|
#include "arange.cuh"
|
||||||
|
|
||||||
static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {
|
static __global__ void arange_f32(float * dst, const int ne0, const float start, const float step) {
|
||||||
// blockIDx.x: idx of ne0 / BLOCK_SIZE
|
// blockIDx.x: idx of ne0 / BLOCK_SIZE
|
||||||
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
if (nidx >= ne0) {
|
if (nidx >= ne0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
dst[nidx] = start + step * nidx;
|
dst[nidx] = start + step * nidx;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {
|
static void arange_f32_cuda(float * dst, const int ne0, const float start, const float step, cudaStream_t stream) {
|
||||||
int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;
|
int num_blocks = (ne0 + CUDA_ARANGE_BLOCK_SIZE - 1) / CUDA_ARANGE_BLOCK_SIZE;
|
||||||
arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);
|
arange_f32<<<num_blocks, CUDA_ARANGE_BLOCK_SIZE, 0, stream>>>(dst, ne0, start, step);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
float start;
|
float start;
|
||||||
float stop;
|
float stop;
|
||||||
float step;
|
float step;
|
||||||
memcpy(&start, (float *)dst->op_params + 0, sizeof(float));
|
memcpy(&start, (float *)dst->op_params + 0, sizeof(float));
|
||||||
memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));
|
memcpy(&stop, (float *)dst->op_params + 1, sizeof(float));
|
||||||
memcpy(&step, (float *)dst->op_params + 2, sizeof(float));
|
memcpy(&step, (float *)dst->op_params + 2, sizeof(float));
|
||||||
|
|
||||||
int64_t steps = (int64_t)ceil((stop - start) / step);
|
int64_t steps = (int64_t)ceil((stop - start) / step);
|
||||||
GGML_ASSERT(ggml_nelements(dst) == steps);
|
GGML_ASSERT(ggml_nelements(dst) == steps);
|
||||||
|
|
||||||
arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);
|
arange_f32_cuda(dst_d, dst->ne[0], start, step, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/arange.cuh
vendored
10
llama/ggml-cuda/arange.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_ARANGE_BLOCK_SIZE 256
|
#define CUDA_ARANGE_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_arange(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
206
llama/ggml-cuda/argsort.cu
vendored
206
llama/ggml-cuda/argsort.cu
vendored
@ -1,103 +1,103 @@
|
|||||||
#include "argsort.cuh"
|
#include "argsort.cuh"
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
static inline __device__ void ggml_cuda_swap(T & a, T & b) {
|
static inline __device__ void ggml_cuda_swap(T & a, T & b) {
|
||||||
T tmp = a;
|
T tmp = a;
|
||||||
a = b;
|
a = b;
|
||||||
b = tmp;
|
b = tmp;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<ggml_sort_order order>
|
template<ggml_sort_order order>
|
||||||
static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {
|
static __global__ void k_argsort_f32_i32(const float * x, int * dst, const int ncols, int ncols_pad) {
|
||||||
// bitonic sort
|
// bitonic sort
|
||||||
int col = threadIdx.x;
|
int col = threadIdx.x;
|
||||||
int row = blockIdx.y;
|
int row = blockIdx.y;
|
||||||
|
|
||||||
if (col >= ncols_pad) {
|
if (col >= ncols_pad) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const float * x_row = x + row * ncols;
|
const float * x_row = x + row * ncols;
|
||||||
extern __shared__ int dst_row[];
|
extern __shared__ int dst_row[];
|
||||||
|
|
||||||
// initialize indices
|
// initialize indices
|
||||||
dst_row[col] = col;
|
dst_row[col] = col;
|
||||||
|
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
|
|
||||||
for (int k = 2; k <= ncols_pad; k *= 2) {
|
for (int k = 2; k <= ncols_pad; k *= 2) {
|
||||||
for (int j = k / 2; j > 0; j /= 2) {
|
for (int j = k / 2; j > 0; j /= 2) {
|
||||||
int ixj = col ^ j;
|
int ixj = col ^ j;
|
||||||
if (ixj > col) {
|
if (ixj > col) {
|
||||||
if ((col & k) == 0) {
|
if ((col & k) == 0) {
|
||||||
if (dst_row[col] >= ncols ||
|
if (dst_row[col] >= ncols ||
|
||||||
(dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?
|
(dst_row[ixj] < ncols && (order == GGML_SORT_ORDER_ASC ?
|
||||||
x_row[dst_row[col]] > x_row[dst_row[ixj]] :
|
x_row[dst_row[col]] > x_row[dst_row[ixj]] :
|
||||||
x_row[dst_row[col]] < x_row[dst_row[ixj]]))
|
x_row[dst_row[col]] < x_row[dst_row[ixj]]))
|
||||||
) {
|
) {
|
||||||
ggml_cuda_swap(dst_row[col], dst_row[ixj]);
|
ggml_cuda_swap(dst_row[col], dst_row[ixj]);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (dst_row[ixj] >= ncols ||
|
if (dst_row[ixj] >= ncols ||
|
||||||
(dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?
|
(dst_row[col] < ncols && (order == GGML_SORT_ORDER_ASC ?
|
||||||
x_row[dst_row[col]] < x_row[dst_row[ixj]] :
|
x_row[dst_row[col]] < x_row[dst_row[ixj]] :
|
||||||
x_row[dst_row[col]] > x_row[dst_row[ixj]]))
|
x_row[dst_row[col]] > x_row[dst_row[ixj]]))
|
||||||
) {
|
) {
|
||||||
ggml_cuda_swap(dst_row[col], dst_row[ixj]);
|
ggml_cuda_swap(dst_row[col], dst_row[ixj]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// copy the result to dst without the padding
|
// copy the result to dst without the padding
|
||||||
if (col < ncols) {
|
if (col < ncols) {
|
||||||
dst[row * ncols + col] = dst_row[col];
|
dst[row * ncols + col] = dst_row[col];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static int next_power_of_2(int x) {
|
static int next_power_of_2(int x) {
|
||||||
int n = 1;
|
int n = 1;
|
||||||
while (n < x) {
|
while (n < x) {
|
||||||
n *= 2;
|
n *= 2;
|
||||||
}
|
}
|
||||||
return n;
|
return n;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {
|
static void argsort_f32_i32_cuda(const float * x, int * dst, const int ncols, const int nrows, ggml_sort_order order, cudaStream_t stream) {
|
||||||
// bitonic sort requires ncols to be power of 2
|
// bitonic sort requires ncols to be power of 2
|
||||||
const int ncols_pad = next_power_of_2(ncols);
|
const int ncols_pad = next_power_of_2(ncols);
|
||||||
|
|
||||||
const dim3 block_dims(ncols_pad, 1, 1);
|
const dim3 block_dims(ncols_pad, 1, 1);
|
||||||
const dim3 block_nums(1, nrows, 1);
|
const dim3 block_nums(1, nrows, 1);
|
||||||
const size_t shared_mem = ncols_pad * sizeof(int);
|
const size_t shared_mem = ncols_pad * sizeof(int);
|
||||||
|
|
||||||
GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
|
GGML_ASSERT(shared_mem <= ggml_cuda_info().devices[ggml_cuda_get_device()].smpb);
|
||||||
|
|
||||||
if (order == GGML_SORT_ORDER_ASC) {
|
if (order == GGML_SORT_ORDER_ASC) {
|
||||||
k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
|
k_argsort_f32_i32<GGML_SORT_ORDER_ASC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
|
||||||
} else if (order == GGML_SORT_ORDER_DESC) {
|
} else if (order == GGML_SORT_ORDER_DESC) {
|
||||||
k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
|
k_argsort_f32_i32<GGML_SORT_ORDER_DESC><<<block_nums, block_dims, shared_mem, stream>>>(x, dst, ncols, ncols_pad);
|
||||||
} else {
|
} else {
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_I32);
|
GGML_ASSERT( dst->type == GGML_TYPE_I32);
|
||||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||||
|
|
||||||
const int64_t ncols = src0->ne[0];
|
const int64_t ncols = src0->ne[0];
|
||||||
const int64_t nrows = ggml_nrows(src0);
|
const int64_t nrows = ggml_nrows(src0);
|
||||||
|
|
||||||
enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
|
enum ggml_sort_order order = (enum ggml_sort_order) dst->op_params[0];
|
||||||
|
|
||||||
argsort_f32_i32_cuda(src0_d, (int *)dst_d, ncols, nrows, order, stream);
|
argsort_f32_i32_cuda(src0_d, (int *)dst_d, ncols, nrows, order, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
6
llama/ggml-cuda/argsort.cuh
vendored
6
llama/ggml-cuda/argsort.cuh
vendored
@ -1,3 +1,3 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_argsort(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
560
llama/ggml-cuda/binbcast.cu
vendored
560
llama/ggml-cuda/binbcast.cu
vendored
@ -1,280 +1,280 @@
|
|||||||
#include "binbcast.cuh"
|
#include "binbcast.cuh"
|
||||||
|
|
||||||
static __device__ __forceinline__ float op_repeat(const float a, const float b) {
|
static __device__ __forceinline__ float op_repeat(const float a, const float b) {
|
||||||
return b;
|
return b;
|
||||||
GGML_UNUSED(a);
|
GGML_UNUSED(a);
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ float op_add(const float a, const float b) {
|
static __device__ __forceinline__ float op_add(const float a, const float b) {
|
||||||
return a + b;
|
return a + b;
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ float op_mul(const float a, const float b) {
|
static __device__ __forceinline__ float op_mul(const float a, const float b) {
|
||||||
return a * b;
|
return a * b;
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ float op_div(const float a, const float b) {
|
static __device__ __forceinline__ float op_div(const float a, const float b) {
|
||||||
return a / b;
|
return a / b;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
|
template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
|
||||||
static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,
|
static __global__ void k_bin_bcast(const src0_t * src0, const src1_t * src1, dst_t * dst,
|
||||||
int ne0, int ne1, int ne2, int ne3,
|
int ne0, int ne1, int ne2, int ne3,
|
||||||
int ne10, int ne11, int ne12, int ne13,
|
int ne10, int ne11, int ne12, int ne13,
|
||||||
/*int s0, */ int s1, int s2, int s3,
|
/*int s0, */ int s1, int s2, int s3,
|
||||||
/*int s00,*/ int s01, int s02, int s03,
|
/*int s00,*/ int s01, int s02, int s03,
|
||||||
/*int s10,*/ int s11, int s12, int s13) {
|
/*int s10,*/ int s11, int s12, int s13) {
|
||||||
const int i0s = blockDim.x*blockIdx.x + threadIdx.x;
|
const int i0s = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);
|
const int i1 = (blockDim.y*blockIdx.y + threadIdx.y);
|
||||||
const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;
|
const int i2 = (blockDim.z*blockIdx.z + threadIdx.z) / ne3;
|
||||||
const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;
|
const int i3 = (blockDim.z*blockIdx.z + threadIdx.z) % ne3;
|
||||||
|
|
||||||
if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
|
if (i0s >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i11 = i1 % ne11;
|
const int i11 = i1 % ne11;
|
||||||
const int i12 = i2 % ne12;
|
const int i12 = i2 % ne12;
|
||||||
const int i13 = i3 % ne13;
|
const int i13 = i3 % ne13;
|
||||||
|
|
||||||
const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
|
const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
|
||||||
const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
|
const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
|
||||||
const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
|
const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
|
||||||
|
|
||||||
const src0_t * src0_row = src0 + i_src0;
|
const src0_t * src0_row = src0 + i_src0;
|
||||||
const src1_t * src1_row = src1 + i_src1;
|
const src1_t * src1_row = src1 + i_src1;
|
||||||
dst_t * dst_row = dst + i_dst;
|
dst_t * dst_row = dst + i_dst;
|
||||||
|
|
||||||
for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {
|
for (int i0 = i0s; i0 < ne0; i0 += blockDim.x*gridDim.x) {
|
||||||
const int i10 = i0 % ne10;
|
const int i10 = i0 % ne10;
|
||||||
dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
|
dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
|
template<float (*bin_op)(const float, const float), typename src0_t, typename src1_t, typename dst_t>
|
||||||
static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,
|
static __global__ void k_bin_bcast_unravel(const src0_t * src0, const src1_t * src1, dst_t * dst,
|
||||||
int ne0, int ne1, int ne2, int ne3,
|
int ne0, int ne1, int ne2, int ne3,
|
||||||
int ne10, int ne11, int ne12, int ne13,
|
int ne10, int ne11, int ne12, int ne13,
|
||||||
/*int s0, */ int s1, int s2, int s3,
|
/*int s0, */ int s1, int s2, int s3,
|
||||||
/*int s00,*/ int s01, int s02, int s03,
|
/*int s00,*/ int s01, int s02, int s03,
|
||||||
/*int s10,*/ int s11, int s12, int s13) {
|
/*int s10,*/ int s11, int s12, int s13) {
|
||||||
|
|
||||||
const int i = blockDim.x*blockIdx.x + threadIdx.x;
|
const int i = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
|
|
||||||
const int i3 = i/(ne2*ne1*ne0);
|
const int i3 = i/(ne2*ne1*ne0);
|
||||||
const int i2 = (i/(ne1*ne0)) % ne2;
|
const int i2 = (i/(ne1*ne0)) % ne2;
|
||||||
const int i1 = (i/ne0) % ne1;
|
const int i1 = (i/ne0) % ne1;
|
||||||
const int i0 = i % ne0;
|
const int i0 = i % ne0;
|
||||||
|
|
||||||
if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
|
if (i0 >= ne0 || i1 >= ne1 || i2 >= ne2 || i3 >= ne3) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i11 = i1 % ne11;
|
const int i11 = i1 % ne11;
|
||||||
const int i12 = i2 % ne12;
|
const int i12 = i2 % ne12;
|
||||||
const int i13 = i3 % ne13;
|
const int i13 = i3 % ne13;
|
||||||
|
|
||||||
const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
|
const size_t i_src0 = i3*s03 + i2*s02 + i1*s01;
|
||||||
const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
|
const size_t i_src1 = i13*s13 + i12*s12 + i11*s11;
|
||||||
const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
|
const size_t i_dst = i3*s3 + i2*s2 + i1*s1;
|
||||||
|
|
||||||
const src0_t * src0_row = src0 + i_src0;
|
const src0_t * src0_row = src0 + i_src0;
|
||||||
const src1_t * src1_row = src1 + i_src1;
|
const src1_t * src1_row = src1 + i_src1;
|
||||||
dst_t * dst_row = dst + i_dst;
|
dst_t * dst_row = dst + i_dst;
|
||||||
|
|
||||||
const int i10 = i0 % ne10;
|
const int i10 = i0 % ne10;
|
||||||
dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
|
dst_row[i0] = (dst_t)bin_op(src0 ? (float)src0_row[i0] : 0.0f, (float)src1_row[i10]);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<float (*bin_op)(const float, const float)>
|
template<float (*bin_op)(const float, const float)>
|
||||||
struct bin_bcast_cuda {
|
struct bin_bcast_cuda {
|
||||||
template<typename src0_t, typename src1_t, typename dst_t>
|
template<typename src0_t, typename src1_t, typename dst_t>
|
||||||
void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,
|
void operator()(const struct ggml_tensor * src0, const struct ggml_tensor * src1, struct ggml_tensor * dst,
|
||||||
const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,
|
const src0_t * src0_dd, const src1_t * src1_dd, dst_t * dst_dd,
|
||||||
cudaStream_t stream) {
|
cudaStream_t stream) {
|
||||||
|
|
||||||
GGML_TENSOR_BINARY_OP_LOCALS
|
GGML_TENSOR_BINARY_OP_LOCALS
|
||||||
|
|
||||||
int nr0 = ne10/ne0;
|
int nr0 = ne10/ne0;
|
||||||
int nr1 = ne11/ne1;
|
int nr1 = ne11/ne1;
|
||||||
int nr2 = ne12/ne2;
|
int nr2 = ne12/ne2;
|
||||||
int nr3 = ne13/ne3;
|
int nr3 = ne13/ne3;
|
||||||
|
|
||||||
int nr[4] = { nr0, nr1, nr2, nr3 };
|
int nr[4] = { nr0, nr1, nr2, nr3 };
|
||||||
|
|
||||||
// collapse dimensions until first broadcast dimension
|
// collapse dimensions until first broadcast dimension
|
||||||
int64_t cne[] = {ne0, ne1, ne2, ne3};
|
int64_t cne[] = {ne0, ne1, ne2, ne3};
|
||||||
int64_t cne0[] = {ne00, ne01, ne02, ne03};
|
int64_t cne0[] = {ne00, ne01, ne02, ne03};
|
||||||
int64_t cne1[] = {ne10, ne11, ne12, ne13};
|
int64_t cne1[] = {ne10, ne11, ne12, ne13};
|
||||||
|
|
||||||
size_t cnb[] = {nb0, nb1, nb2, nb3};
|
size_t cnb[] = {nb0, nb1, nb2, nb3};
|
||||||
size_t cnb0[] = {nb00, nb01, nb02, nb03};
|
size_t cnb0[] = {nb00, nb01, nb02, nb03};
|
||||||
size_t cnb1[] = {nb10, nb11, nb12, nb13};
|
size_t cnb1[] = {nb10, nb11, nb12, nb13};
|
||||||
|
|
||||||
auto collapse = [](int64_t cne[]) {
|
auto collapse = [](int64_t cne[]) {
|
||||||
cne[0] *= cne[1];
|
cne[0] *= cne[1];
|
||||||
cne[1] = cne[2];
|
cne[1] = cne[2];
|
||||||
cne[2] = cne[3];
|
cne[2] = cne[3];
|
||||||
cne[3] = 1;
|
cne[3] = 1;
|
||||||
};
|
};
|
||||||
|
|
||||||
auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {
|
auto collapse_nb = [](size_t cnb[], const int64_t cne[]) {
|
||||||
cnb[1] *= cne[1];
|
cnb[1] *= cne[1];
|
||||||
cnb[2] *= cne[2];
|
cnb[2] *= cne[2];
|
||||||
cnb[3] *= cne[3];
|
cnb[3] *= cne[3];
|
||||||
};
|
};
|
||||||
|
|
||||||
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {
|
if (ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(dst)) {
|
||||||
for (int i = 0; i < 4; i++) {
|
for (int i = 0; i < 4; i++) {
|
||||||
if (nr[i] != 1) {
|
if (nr[i] != 1) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
if (i > 0) {
|
if (i > 0) {
|
||||||
collapse_nb(cnb, cne);
|
collapse_nb(cnb, cne);
|
||||||
collapse_nb(cnb0, cne0);
|
collapse_nb(cnb0, cne0);
|
||||||
collapse_nb(cnb1, cne1);
|
collapse_nb(cnb1, cne1);
|
||||||
collapse(cne);
|
collapse(cne);
|
||||||
collapse(cne0);
|
collapse(cne0);
|
||||||
collapse(cne1);
|
collapse(cne1);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
int64_t ne0 = cne[0];
|
int64_t ne0 = cne[0];
|
||||||
int64_t ne1 = cne[1];
|
int64_t ne1 = cne[1];
|
||||||
int64_t ne2 = cne[2];
|
int64_t ne2 = cne[2];
|
||||||
int64_t ne3 = cne[3];
|
int64_t ne3 = cne[3];
|
||||||
|
|
||||||
//int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);
|
//int64_t ne00 = cne0[0]; GGML_UNUSED(ne00);
|
||||||
//int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);
|
//int64_t ne01 = cne0[1]; GGML_UNUSED(ne01);
|
||||||
//int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);
|
//int64_t ne02 = cne0[2]; GGML_UNUSED(ne02);
|
||||||
//int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);
|
//int64_t ne03 = cne0[3]; GGML_UNUSED(ne03);
|
||||||
|
|
||||||
int64_t ne10 = cne1[0];
|
int64_t ne10 = cne1[0];
|
||||||
int64_t ne11 = cne1[1];
|
int64_t ne11 = cne1[1];
|
||||||
int64_t ne12 = cne1[2];
|
int64_t ne12 = cne1[2];
|
||||||
int64_t ne13 = cne1[3];
|
int64_t ne13 = cne1[3];
|
||||||
|
|
||||||
size_t nb0 = cnb[0];
|
size_t nb0 = cnb[0];
|
||||||
size_t nb1 = cnb[1];
|
size_t nb1 = cnb[1];
|
||||||
size_t nb2 = cnb[2];
|
size_t nb2 = cnb[2];
|
||||||
size_t nb3 = cnb[3];
|
size_t nb3 = cnb[3];
|
||||||
|
|
||||||
size_t nb00 = cnb0[0];
|
size_t nb00 = cnb0[0];
|
||||||
size_t nb01 = cnb0[1];
|
size_t nb01 = cnb0[1];
|
||||||
size_t nb02 = cnb0[2];
|
size_t nb02 = cnb0[2];
|
||||||
size_t nb03 = cnb0[3];
|
size_t nb03 = cnb0[3];
|
||||||
|
|
||||||
size_t nb10 = cnb1[0];
|
size_t nb10 = cnb1[0];
|
||||||
size_t nb11 = cnb1[1];
|
size_t nb11 = cnb1[1];
|
||||||
size_t nb12 = cnb1[2];
|
size_t nb12 = cnb1[2];
|
||||||
size_t nb13 = cnb1[3];
|
size_t nb13 = cnb1[3];
|
||||||
|
|
||||||
size_t s0 = nb0 / sizeof(dst_t);
|
size_t s0 = nb0 / sizeof(dst_t);
|
||||||
size_t s1 = nb1 / sizeof(dst_t);
|
size_t s1 = nb1 / sizeof(dst_t);
|
||||||
size_t s2 = nb2 / sizeof(dst_t);
|
size_t s2 = nb2 / sizeof(dst_t);
|
||||||
size_t s3 = nb3 / sizeof(dst_t);
|
size_t s3 = nb3 / sizeof(dst_t);
|
||||||
|
|
||||||
size_t s10 = nb10 / sizeof(src1_t);
|
size_t s10 = nb10 / sizeof(src1_t);
|
||||||
size_t s11 = nb11 / sizeof(src1_t);
|
size_t s11 = nb11 / sizeof(src1_t);
|
||||||
size_t s12 = nb12 / sizeof(src1_t);
|
size_t s12 = nb12 / sizeof(src1_t);
|
||||||
size_t s13 = nb13 / sizeof(src1_t);
|
size_t s13 = nb13 / sizeof(src1_t);
|
||||||
|
|
||||||
size_t s00 = nb00 / sizeof(src0_t);
|
size_t s00 = nb00 / sizeof(src0_t);
|
||||||
size_t s01 = nb01 / sizeof(src0_t);
|
size_t s01 = nb01 / sizeof(src0_t);
|
||||||
size_t s02 = nb02 / sizeof(src0_t);
|
size_t s02 = nb02 / sizeof(src0_t);
|
||||||
size_t s03 = nb03 / sizeof(src0_t);
|
size_t s03 = nb03 / sizeof(src0_t);
|
||||||
|
|
||||||
GGML_ASSERT(nb0 % sizeof(dst_t) == 0);
|
GGML_ASSERT(nb0 % sizeof(dst_t) == 0);
|
||||||
GGML_ASSERT(nb1 % sizeof(dst_t) == 0);
|
GGML_ASSERT(nb1 % sizeof(dst_t) == 0);
|
||||||
GGML_ASSERT(nb2 % sizeof(dst_t) == 0);
|
GGML_ASSERT(nb2 % sizeof(dst_t) == 0);
|
||||||
GGML_ASSERT(nb3 % sizeof(dst_t) == 0);
|
GGML_ASSERT(nb3 % sizeof(dst_t) == 0);
|
||||||
|
|
||||||
GGML_ASSERT(nb00 % sizeof(src0_t) == 0);
|
GGML_ASSERT(nb00 % sizeof(src0_t) == 0);
|
||||||
GGML_ASSERT(nb01 % sizeof(src0_t) == 0);
|
GGML_ASSERT(nb01 % sizeof(src0_t) == 0);
|
||||||
GGML_ASSERT(nb02 % sizeof(src0_t) == 0);
|
GGML_ASSERT(nb02 % sizeof(src0_t) == 0);
|
||||||
GGML_ASSERT(nb03 % sizeof(src0_t) == 0);
|
GGML_ASSERT(nb03 % sizeof(src0_t) == 0);
|
||||||
|
|
||||||
GGML_ASSERT(nb10 % sizeof(src1_t) == 0);
|
GGML_ASSERT(nb10 % sizeof(src1_t) == 0);
|
||||||
GGML_ASSERT(nb11 % sizeof(src1_t) == 0);
|
GGML_ASSERT(nb11 % sizeof(src1_t) == 0);
|
||||||
GGML_ASSERT(nb12 % sizeof(src1_t) == 0);
|
GGML_ASSERT(nb12 % sizeof(src1_t) == 0);
|
||||||
GGML_ASSERT(nb13 % sizeof(src1_t) == 0);
|
GGML_ASSERT(nb13 % sizeof(src1_t) == 0);
|
||||||
|
|
||||||
GGML_ASSERT(s0 == 1);
|
GGML_ASSERT(s0 == 1);
|
||||||
GGML_ASSERT(s00 == 1);
|
GGML_ASSERT(s00 == 1);
|
||||||
GGML_ASSERT(s10 == 1);
|
GGML_ASSERT(s10 == 1);
|
||||||
|
|
||||||
const int block_size = 128;
|
const int block_size = 128;
|
||||||
|
|
||||||
int64_t hne0 = std::max(ne0/2LL, 1LL);
|
int64_t hne0 = std::max(ne0/2LL, 1LL);
|
||||||
|
|
||||||
dim3 block_dims;
|
dim3 block_dims;
|
||||||
block_dims.x = std::min<unsigned int>(hne0, block_size);
|
block_dims.x = std::min<unsigned int>(hne0, block_size);
|
||||||
block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);
|
block_dims.y = std::min<unsigned int>(ne1, block_size / block_dims.x);
|
||||||
block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);
|
block_dims.z = std::min(std::min<unsigned int>(ne2*ne3, block_size / block_dims.x / block_dims.y), 64U);
|
||||||
|
|
||||||
dim3 block_nums(
|
dim3 block_nums(
|
||||||
(hne0 + block_dims.x - 1) / block_dims.x,
|
(hne0 + block_dims.x - 1) / block_dims.x,
|
||||||
(ne1 + block_dims.y - 1) / block_dims.y,
|
(ne1 + block_dims.y - 1) / block_dims.y,
|
||||||
(ne2*ne3 + block_dims.z - 1) / block_dims.z
|
(ne2*ne3 + block_dims.z - 1) / block_dims.z
|
||||||
);
|
);
|
||||||
|
|
||||||
if (block_nums.z > 65535) {
|
if (block_nums.z > 65535) {
|
||||||
// this is the maximum number of blocks in z dimension, fallback to 1D grid kernel
|
// this is the maximum number of blocks in z dimension, fallback to 1D grid kernel
|
||||||
int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;
|
int block_num = (ne0*ne1*ne2*ne3 + block_size - 1) / block_size;
|
||||||
k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(
|
k_bin_bcast_unravel<bin_op><<<block_num, block_size, 0, stream>>>(
|
||||||
src0_dd, src1_dd, dst_dd,
|
src0_dd, src1_dd, dst_dd,
|
||||||
ne0, ne1, ne2, ne3,
|
ne0, ne1, ne2, ne3,
|
||||||
ne10, ne11, ne12, ne13,
|
ne10, ne11, ne12, ne13,
|
||||||
/* s0, */ s1, s2, s3,
|
/* s0, */ s1, s2, s3,
|
||||||
/* s00, */ s01, s02, s03,
|
/* s00, */ s01, s02, s03,
|
||||||
/* s10, */ s11, s12, s13);
|
/* s10, */ s11, s12, s13);
|
||||||
} else {
|
} else {
|
||||||
k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(
|
k_bin_bcast<bin_op><<<block_nums, block_dims, 0, stream>>>(
|
||||||
src0_dd, src1_dd, dst_dd,
|
src0_dd, src1_dd, dst_dd,
|
||||||
ne0, ne1, ne2, ne3,
|
ne0, ne1, ne2, ne3,
|
||||||
ne10, ne11, ne12, ne13,
|
ne10, ne11, ne12, ne13,
|
||||||
/* s0, */ s1, s2, s3,
|
/* s0, */ s1, s2, s3,
|
||||||
/* s00, */ s01, s02, s03,
|
/* s00, */ s01, s02, s03,
|
||||||
/* s10, */ s11, s12, s13);
|
/* s10, */ s11, s12, s13);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
template<class op>
|
template<class op>
|
||||||
static void ggml_cuda_op_bin_bcast(
|
static void ggml_cuda_op_bin_bcast(
|
||||||
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||||
const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {
|
const void * src0_dd, const void * src1_dd, void * dst_dd, cudaStream_t stream) {
|
||||||
|
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
if (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
|
||||||
op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
|
op()(src0, src1, dst, (const float *)src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
|
||||||
} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {
|
} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) {
|
||||||
op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);
|
op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (half *) dst_dd, stream);
|
||||||
} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {
|
} else if (src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) {
|
||||||
op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
|
op()(src0, src1, dst, (const half *) src0_dd, (const float *)src1_dd, (float *)dst_dd, stream);
|
||||||
} else {
|
} else {
|
||||||
fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
|
fprintf(stderr, "%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
|
||||||
ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));
|
ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());
|
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_repeat>>(dst, dst->src[0], dst, nullptr, dst->src[0]->data, dst->data, ctx.stream());
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_add>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_mul>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
ggml_cuda_op_bin_bcast<bin_bcast_cuda<op_div>>(dst->src[0], dst->src[1], dst, dst->src[0]->data, dst->src[1]->data, dst->data, ctx.stream());
|
||||||
}
|
}
|
||||||
|
|||||||
12
llama/ggml-cuda/binbcast.cuh
vendored
12
llama/ggml-cuda/binbcast.cuh
vendored
@ -1,6 +1,6 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_repeat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_add(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_mul(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_div(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
10
llama/ggml-cuda/clamp.cuh
vendored
10
llama/ggml-cuda/clamp.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_CLAMP_BLOCK_SIZE 256
|
#define CUDA_CLAMP_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_clamp(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
98
llama/ggml-cuda/concat.cu
vendored
98
llama/ggml-cuda/concat.cu
vendored
@ -1,49 +1,49 @@
|
|||||||
#include "concat.cuh"
|
#include "concat.cuh"
|
||||||
|
|
||||||
static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {
|
static __global__ void concat_f32(const float * x,const float * y, float * dst, const int ne0, const int ne02) {
|
||||||
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
if (nidx >= ne0) {
|
if (nidx >= ne0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// operation
|
// operation
|
||||||
int offset_dst =
|
int offset_dst =
|
||||||
nidx +
|
nidx +
|
||||||
blockIdx.y * ne0 +
|
blockIdx.y * ne0 +
|
||||||
blockIdx.z * ne0 * gridDim.y;
|
blockIdx.z * ne0 * gridDim.y;
|
||||||
if (blockIdx.z < ne02) { // src0
|
if (blockIdx.z < ne02) { // src0
|
||||||
int offset_src =
|
int offset_src =
|
||||||
nidx +
|
nidx +
|
||||||
blockIdx.y * ne0 +
|
blockIdx.y * ne0 +
|
||||||
blockIdx.z * ne0 * gridDim.y;
|
blockIdx.z * ne0 * gridDim.y;
|
||||||
dst[offset_dst] = x[offset_src];
|
dst[offset_dst] = x[offset_src];
|
||||||
} else {
|
} else {
|
||||||
int offset_src =
|
int offset_src =
|
||||||
nidx +
|
nidx +
|
||||||
blockIdx.y * ne0 +
|
blockIdx.y * ne0 +
|
||||||
(blockIdx.z - ne02) * ne0 * gridDim.y;
|
(blockIdx.z - ne02) * ne0 * gridDim.y;
|
||||||
dst[offset_dst] = y[offset_src];
|
dst[offset_dst] = y[offset_src];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {
|
static void concat_f32_cuda(const float * x, const float * y, float * dst, const int ne0, int ne1, int ne2, int ne02, cudaStream_t stream) {
|
||||||
int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;
|
int num_blocks = (ne0 + CUDA_CONCAT_BLOCK_SIZE - 1) / CUDA_CONCAT_BLOCK_SIZE;
|
||||||
dim3 gridDim(num_blocks, ne1, ne2);
|
dim3 gridDim(num_blocks, ne1, ne2);
|
||||||
concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);
|
concat_f32<<<gridDim, CUDA_CONCAT_BLOCK_SIZE, 0, stream>>>(x, y, dst, ne0, ne02);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const ggml_tensor * src1 = dst->src[1];
|
const ggml_tensor * src1 = dst->src[1];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
const float * src1_d = (const float *)src1->data;
|
const float * src1_d = (const float *)src1->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
|
for (int i3 = 0; i3 < dst->ne[3]; i3++) {
|
||||||
concat_f32_cuda(src0_d + i3 * (src0->nb[3] / 4), src1_d + i3 * (src1->nb[3] / 4), dst_d + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], stream);
|
concat_f32_cuda(src0_d + i3 * (src0->nb[3] / 4), src1_d + i3 * (src1->nb[3] / 4), dst_d + i3 * (dst->nb[3] / 4), dst->ne[0], dst->ne[1], dst->ne[2], src0->ne[2], stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/concat.cuh
vendored
10
llama/ggml-cuda/concat.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_CONCAT_BLOCK_SIZE 256
|
#define CUDA_CONCAT_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_concat(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
26
llama/ggml-cuda/convert.cuh
vendored
26
llama/ggml-cuda/convert.cuh
vendored
@ -1,13 +1,13 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
|
#define CUDA_DEQUANTIZE_BLOCK_SIZE 256
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, cudaStream_t stream);
|
using to_t_cuda_t = void (*)(const void * __restrict__ x, T * __restrict__ y, int64_t k, cudaStream_t stream);
|
||||||
|
|
||||||
typedef to_t_cuda_t<float> to_fp32_cuda_t;
|
typedef to_t_cuda_t<float> to_fp32_cuda_t;
|
||||||
typedef to_t_cuda_t<half> to_fp16_cuda_t;
|
typedef to_t_cuda_t<half> to_fp16_cuda_t;
|
||||||
|
|
||||||
to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);
|
to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type);
|
||||||
|
|
||||||
to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);
|
to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type);
|
||||||
|
|||||||
206
llama/ggml-cuda/dequantize.cuh
vendored
206
llama/ggml-cuda/dequantize.cuh
vendored
@ -1,103 +1,103 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
static __device__ __forceinline__ void dequantize_q4_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||||
const block_q4_0 * x = (const block_q4_0 *) vx;
|
const block_q4_0 * x = (const block_q4_0 *) vx;
|
||||||
|
|
||||||
const dfloat d = x[ib].d;
|
const dfloat d = x[ib].d;
|
||||||
|
|
||||||
const int vui = x[ib].qs[iqs];
|
const int vui = x[ib].qs[iqs];
|
||||||
|
|
||||||
v.x = vui & 0xF;
|
v.x = vui & 0xF;
|
||||||
v.y = vui >> 4;
|
v.y = vui >> 4;
|
||||||
|
|
||||||
#ifdef GGML_CUDA_F16
|
#ifdef GGML_CUDA_F16
|
||||||
v = __hsub2(v, {8.0f, 8.0f});
|
v = __hsub2(v, {8.0f, 8.0f});
|
||||||
v = __hmul2(v, {d, d});
|
v = __hmul2(v, {d, d});
|
||||||
#else
|
#else
|
||||||
v.x = (v.x - 8.0f) * d;
|
v.x = (v.x - 8.0f) * d;
|
||||||
v.y = (v.y - 8.0f) * d;
|
v.y = (v.y - 8.0f) * d;
|
||||||
#endif // GGML_CUDA_F16
|
#endif // GGML_CUDA_F16
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
static __device__ __forceinline__ void dequantize_q4_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||||
const block_q4_1 * x = (const block_q4_1 *) vx;
|
const block_q4_1 * x = (const block_q4_1 *) vx;
|
||||||
|
|
||||||
const dfloat d = __low2half(x[ib].dm);
|
const dfloat d = __low2half(x[ib].dm);
|
||||||
const dfloat m = __high2half(x[ib].dm);
|
const dfloat m = __high2half(x[ib].dm);
|
||||||
|
|
||||||
const int vui = x[ib].qs[iqs];
|
const int vui = x[ib].qs[iqs];
|
||||||
|
|
||||||
v.x = vui & 0xF;
|
v.x = vui & 0xF;
|
||||||
v.y = vui >> 4;
|
v.y = vui >> 4;
|
||||||
|
|
||||||
#ifdef GGML_CUDA_F16
|
#ifdef GGML_CUDA_F16
|
||||||
v = __hmul2(v, {d, d});
|
v = __hmul2(v, {d, d});
|
||||||
v = __hadd2(v, {m, m});
|
v = __hadd2(v, {m, m});
|
||||||
#else
|
#else
|
||||||
v.x = (v.x * d) + m;
|
v.x = (v.x * d) + m;
|
||||||
v.y = (v.y * d) + m;
|
v.y = (v.y * d) + m;
|
||||||
#endif // GGML_CUDA_F16
|
#endif // GGML_CUDA_F16
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
static __device__ __forceinline__ void dequantize_q5_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||||
const block_q5_0 * x = (const block_q5_0 *) vx;
|
const block_q5_0 * x = (const block_q5_0 *) vx;
|
||||||
|
|
||||||
const dfloat d = x[ib].d;
|
const dfloat d = x[ib].d;
|
||||||
|
|
||||||
uint32_t qh;
|
uint32_t qh;
|
||||||
memcpy(&qh, x[ib].qh, sizeof(qh));
|
memcpy(&qh, x[ib].qh, sizeof(qh));
|
||||||
|
|
||||||
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
||||||
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
||||||
|
|
||||||
v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
|
v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
|
||||||
v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
|
v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
|
||||||
|
|
||||||
#ifdef GGML_CUDA_F16
|
#ifdef GGML_CUDA_F16
|
||||||
v = __hsub2(v, {16.0f, 16.0f});
|
v = __hsub2(v, {16.0f, 16.0f});
|
||||||
v = __hmul2(v, {d, d});
|
v = __hmul2(v, {d, d});
|
||||||
#else
|
#else
|
||||||
v.x = (v.x - 16.0f) * d;
|
v.x = (v.x - 16.0f) * d;
|
||||||
v.y = (v.y - 16.0f) * d;
|
v.y = (v.y - 16.0f) * d;
|
||||||
#endif // GGML_CUDA_F16
|
#endif // GGML_CUDA_F16
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
static __device__ __forceinline__ void dequantize_q5_1(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||||
const block_q5_1 * x = (const block_q5_1 *) vx;
|
const block_q5_1 * x = (const block_q5_1 *) vx;
|
||||||
|
|
||||||
const dfloat d = __low2half(x[ib].dm);
|
const dfloat d = __low2half(x[ib].dm);
|
||||||
const dfloat m = __high2half(x[ib].dm);
|
const dfloat m = __high2half(x[ib].dm);
|
||||||
|
|
||||||
uint32_t qh;
|
uint32_t qh;
|
||||||
memcpy(&qh, x[ib].qh, sizeof(qh));
|
memcpy(&qh, x[ib].qh, sizeof(qh));
|
||||||
|
|
||||||
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
const int xh_0 = ((qh >> (iqs + 0)) << 4) & 0x10;
|
||||||
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
const int xh_1 = ((qh >> (iqs + 12)) ) & 0x10;
|
||||||
|
|
||||||
v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
|
v.x = ((x[ib].qs[iqs] & 0xf) | xh_0);
|
||||||
v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
|
v.y = ((x[ib].qs[iqs] >> 4) | xh_1);
|
||||||
|
|
||||||
#ifdef GGML_CUDA_F16
|
#ifdef GGML_CUDA_F16
|
||||||
v = __hmul2(v, {d, d});
|
v = __hmul2(v, {d, d});
|
||||||
v = __hadd2(v, {m, m});
|
v = __hadd2(v, {m, m});
|
||||||
#else
|
#else
|
||||||
v.x = (v.x * d) + m;
|
v.x = (v.x * d) + m;
|
||||||
v.y = (v.y * d) + m;
|
v.y = (v.y * d) + m;
|
||||||
#endif // GGML_CUDA_F16
|
#endif // GGML_CUDA_F16
|
||||||
}
|
}
|
||||||
|
|
||||||
static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const int64_t ib, const int iqs, dfloat2 & v){
|
||||||
const block_q8_0 * x = (const block_q8_0 *) vx;
|
const block_q8_0 * x = (const block_q8_0 *) vx;
|
||||||
|
|
||||||
const dfloat d = x[ib].d;
|
const dfloat d = x[ib].d;
|
||||||
|
|
||||||
v.x = x[ib].qs[iqs + 0];
|
v.x = x[ib].qs[iqs + 0];
|
||||||
v.y = x[ib].qs[iqs + 1];
|
v.y = x[ib].qs[iqs + 1];
|
||||||
|
|
||||||
#ifdef GGML_CUDA_F16
|
#ifdef GGML_CUDA_F16
|
||||||
v = __hmul2(v, {d, d});
|
v = __hmul2(v, {d, d});
|
||||||
#else
|
#else
|
||||||
v.x *= d;
|
v.x *= d;
|
||||||
v.y *= d;
|
v.y *= d;
|
||||||
#endif // GGML_CUDA_F16
|
#endif // GGML_CUDA_F16
|
||||||
}
|
}
|
||||||
|
|||||||
80
llama/ggml-cuda/diagmask.cu
vendored
80
llama/ggml-cuda/diagmask.cu
vendored
@ -1,40 +1,40 @@
|
|||||||
#include "diagmask.cuh"
|
#include "diagmask.cuh"
|
||||||
|
|
||||||
static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {
|
static __global__ void diag_mask_inf_f32(const float * x, float * dst, const int ncols, const int rows_per_channel, const int n_past) {
|
||||||
const int col = blockDim.y*blockIdx.y + threadIdx.y;
|
const int col = blockDim.y*blockIdx.y + threadIdx.y;
|
||||||
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
|
|
||||||
if (col >= ncols) {
|
if (col >= ncols) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i = row*ncols + col;
|
const int i = row*ncols + col;
|
||||||
//dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];
|
//dst[i] = col > (n_past + row % rows_per_channel) ? -INFINITY : x[i];
|
||||||
//dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU
|
//dst[i] = x[i] - (col > n_past + row % rows_per_channel) * INT_MAX; // equivalent within rounding error but slightly faster on GPU
|
||||||
dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;
|
dst[i] = x[i] - (col > n_past + row % rows_per_channel) * FLT_MAX;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {
|
static void diag_mask_inf_f32_cuda(const float * x, float * dst, const int ncols_x, const int nrows_x, const int rows_per_channel, const int n_past, cudaStream_t stream) {
|
||||||
const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);
|
const dim3 block_dims(1, CUDA_DIAG_MASK_INF_BLOCK_SIZE, 1);
|
||||||
const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;
|
const int block_num_x = (ncols_x + CUDA_DIAG_MASK_INF_BLOCK_SIZE - 1) / CUDA_DIAG_MASK_INF_BLOCK_SIZE;
|
||||||
const dim3 block_nums(nrows_x, block_num_x, 1);
|
const dim3 block_nums(nrows_x, block_num_x, 1);
|
||||||
diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);
|
diag_mask_inf_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols_x, rows_per_channel, n_past);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int64_t ne00 = src0->ne[0];
|
const int64_t ne00 = src0->ne[0];
|
||||||
const int64_t ne01 = src0->ne[1];
|
const int64_t ne01 = src0->ne[1];
|
||||||
const int nrows0 = ggml_nrows(src0);
|
const int nrows0 = ggml_nrows(src0);
|
||||||
|
|
||||||
const int n_past = ((int32_t *) dst->op_params)[0];
|
const int n_past = ((int32_t *) dst->op_params)[0];
|
||||||
|
|
||||||
diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);
|
diag_mask_inf_f32_cuda(src0_d, dst_d, ne00, nrows0, ne01, n_past, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/diagmask.cuh
vendored
10
llama/ggml-cuda/diagmask.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32
|
#define CUDA_DIAG_MASK_INF_BLOCK_SIZE 32
|
||||||
|
|
||||||
void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_diag_mask_inf(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
1626
llama/ggml-cuda/dmmv.cu
vendored
1626
llama/ggml-cuda/dmmv.cu
vendored
File diff suppressed because it is too large
Load Diff
36
llama/ggml-cuda/dmmv.cuh
vendored
36
llama/ggml-cuda/dmmv.cuh
vendored
@ -1,18 +1,18 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
// dmmv = dequantize_mul_mat_vec
|
// dmmv = dequantize_mul_mat_vec
|
||||||
|
|
||||||
// TODO: remove this?
|
// TODO: remove this?
|
||||||
#ifndef GGML_CUDA_DMMV_X
|
#ifndef GGML_CUDA_DMMV_X
|
||||||
#define GGML_CUDA_DMMV_X 32
|
#define GGML_CUDA_DMMV_X 32
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#ifndef GGML_CUDA_MMV_Y
|
#ifndef GGML_CUDA_MMV_Y
|
||||||
#define GGML_CUDA_MMV_Y 1
|
#define GGML_CUDA_MMV_Y 1
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void ggml_cuda_op_dequantize_mul_mat_vec(
|
void ggml_cuda_op_dequantize_mul_mat_vec(
|
||||||
ggml_backend_cuda_context & ctx,
|
ggml_backend_cuda_context & ctx,
|
||||||
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
||||||
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
||||||
const int64_t src1_padded_row_size, cudaStream_t stream);
|
const int64_t src1_padded_row_size, cudaStream_t stream);
|
||||||
|
|||||||
356
llama/ggml-cuda/getrows.cu
vendored
356
llama/ggml-cuda/getrows.cu
vendored
@ -1,178 +1,178 @@
|
|||||||
#include "getrows.cuh"
|
#include "getrows.cuh"
|
||||||
#include "dequantize.cuh"
|
#include "dequantize.cuh"
|
||||||
|
|
||||||
template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
|
template<int qk, int qr, dequantize_kernel_t dequantize_kernel, typename dst_t>
|
||||||
static __global__ void k_get_rows(
|
static __global__ void k_get_rows(
|
||||||
const void * src0, const int32_t * src1, dst_t * dst,
|
const void * src0, const int32_t * src1, dst_t * dst,
|
||||||
int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
|
int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
|
||||||
/*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
|
/*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
|
||||||
/*size_t s0,*/ size_t s1, size_t s2, size_t s3,
|
/*size_t s0,*/ size_t s1, size_t s2, size_t s3,
|
||||||
/*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
|
/*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
|
||||||
size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
|
size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
|
||||||
|
|
||||||
const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;
|
const int i00 = (blockIdx.x*blockDim.x + threadIdx.x)*2;
|
||||||
const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
|
const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
|
||||||
const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
|
const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
|
||||||
const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
|
const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
|
||||||
|
|
||||||
if (i00 >= ne00) {
|
if (i00 >= ne00) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
|
const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
|
||||||
|
|
||||||
dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
|
dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
|
||||||
const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;
|
const void * src0_row = (const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03;
|
||||||
|
|
||||||
const int ib = i00/qk; // block index
|
const int ib = i00/qk; // block index
|
||||||
const int iqs = (i00%qk)/qr; // quant index
|
const int iqs = (i00%qk)/qr; // quant index
|
||||||
const int iybs = i00 - i00%qk; // dst block start index
|
const int iybs = i00 - i00%qk; // dst block start index
|
||||||
const int y_offset = qr == 1 ? 1 : qk/2;
|
const int y_offset = qr == 1 ? 1 : qk/2;
|
||||||
|
|
||||||
// dequantize
|
// dequantize
|
||||||
dfloat2 v;
|
dfloat2 v;
|
||||||
dequantize_kernel(src0_row, ib, iqs, v);
|
dequantize_kernel(src0_row, ib, iqs, v);
|
||||||
|
|
||||||
dst_row[iybs + iqs + 0] = v.x;
|
dst_row[iybs + iqs + 0] = v.x;
|
||||||
dst_row[iybs + iqs + y_offset] = v.y;
|
dst_row[iybs + iqs + y_offset] = v.y;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename src0_t, typename dst_t>
|
template<typename src0_t, typename dst_t>
|
||||||
static __global__ void k_get_rows_float(
|
static __global__ void k_get_rows_float(
|
||||||
const src0_t * src0, const int32_t * src1, dst_t * dst,
|
const src0_t * src0, const int32_t * src1, dst_t * dst,
|
||||||
int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
|
int64_t ne00, /*int64_t ne01, int64_t ne02, int64_t ne03,*/
|
||||||
/*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
|
/*int64_t ne10, int64_t ne11,*/ int64_t ne12, /*int64_t ne13,*/
|
||||||
/*size_t s0,*/ size_t s1, size_t s2, size_t s3,
|
/*size_t s0,*/ size_t s1, size_t s2, size_t s3,
|
||||||
/*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
|
/*size_t nb00,*/ size_t nb01, size_t nb02, size_t nb03,
|
||||||
size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
|
size_t s10, size_t s11, size_t s12/*, size_t s13*/) {
|
||||||
|
|
||||||
const int i00 = blockIdx.x*blockDim.x + threadIdx.x;
|
const int i00 = blockIdx.x*blockDim.x + threadIdx.x;
|
||||||
const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
|
const int i10 = blockDim.y*blockIdx.y + threadIdx.y;
|
||||||
const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
|
const int i11 = (blockIdx.z*blockDim.z + threadIdx.z)/ne12;
|
||||||
const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
|
const int i12 = (blockIdx.z*blockDim.z + threadIdx.z)%ne12;
|
||||||
|
|
||||||
if (i00 >= ne00) {
|
if (i00 >= ne00) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
|
const int i01 = src1[i10*s10 + i11*s11 + i12*s12];
|
||||||
|
|
||||||
dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
|
dst_t * dst_row = dst + i10*s1 + i11*s2 + i12*s3;
|
||||||
const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);
|
const src0_t * src0_row = (const src0_t *)((const char *)src0 + i01*nb01 + i11*nb02 + i12*nb03);
|
||||||
|
|
||||||
dst_row[i00] = src0_row[i00];
|
dst_row[i00] = src0_row[i00];
|
||||||
}
|
}
|
||||||
|
|
||||||
template<int qk, int qr, dequantize_kernel_t dq>
|
template<int qk, int qr, dequantize_kernel_t dq>
|
||||||
static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
static void get_rows_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||||
const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
|
const void * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
|
||||||
|
|
||||||
GGML_TENSOR_BINARY_OP_LOCALS
|
GGML_TENSOR_BINARY_OP_LOCALS
|
||||||
|
|
||||||
const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
|
const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
|
||||||
const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);
|
const int block_num_x = (ne00 + 2*CUDA_GET_ROWS_BLOCK_SIZE - 1) / (2*CUDA_GET_ROWS_BLOCK_SIZE);
|
||||||
const dim3 block_nums(block_num_x, ne10, ne11*ne12);
|
const dim3 block_nums(block_num_x, ne10, ne11*ne12);
|
||||||
|
|
||||||
// strides in elements
|
// strides in elements
|
||||||
//const size_t s0 = nb0 / ggml_element_size(dst);
|
//const size_t s0 = nb0 / ggml_element_size(dst);
|
||||||
const size_t s1 = nb1 / ggml_element_size(dst);
|
const size_t s1 = nb1 / ggml_element_size(dst);
|
||||||
const size_t s2 = nb2 / ggml_element_size(dst);
|
const size_t s2 = nb2 / ggml_element_size(dst);
|
||||||
const size_t s3 = nb3 / ggml_element_size(dst);
|
const size_t s3 = nb3 / ggml_element_size(dst);
|
||||||
|
|
||||||
const size_t s10 = nb10 / ggml_element_size(src1);
|
const size_t s10 = nb10 / ggml_element_size(src1);
|
||||||
const size_t s11 = nb11 / ggml_element_size(src1);
|
const size_t s11 = nb11 / ggml_element_size(src1);
|
||||||
const size_t s12 = nb12 / ggml_element_size(src1);
|
const size_t s12 = nb12 / ggml_element_size(src1);
|
||||||
//const size_t s13 = nb13 / ggml_element_size(src1);
|
//const size_t s13 = nb13 / ggml_element_size(src1);
|
||||||
|
|
||||||
GGML_ASSERT(ne00 % 2 == 0);
|
GGML_ASSERT(ne00 % 2 == 0);
|
||||||
|
|
||||||
k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(
|
k_get_rows<qk, qr, dq><<<block_nums, block_dims, 0, stream>>>(
|
||||||
src0_dd, src1_dd, dst_dd,
|
src0_dd, src1_dd, dst_dd,
|
||||||
ne00, /*ne01, ne02, ne03,*/
|
ne00, /*ne01, ne02, ne03,*/
|
||||||
/*ne10, ne11,*/ ne12, /*ne13,*/
|
/*ne10, ne11,*/ ne12, /*ne13,*/
|
||||||
/* s0,*/ s1, s2, s3,
|
/* s0,*/ s1, s2, s3,
|
||||||
/* nb00,*/ nb01, nb02, nb03,
|
/* nb00,*/ nb01, nb02, nb03,
|
||||||
s10, s11, s12/*, s13*/);
|
s10, s11, s12/*, s13*/);
|
||||||
|
|
||||||
GGML_UNUSED(dst);
|
GGML_UNUSED(dst);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename src0_t>
|
template<typename src0_t>
|
||||||
static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
static void get_rows_cuda_float(const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst,
|
||||||
const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
|
const src0_t * src0_dd, const int32_t * src1_dd, float * dst_dd, cudaStream_t stream) {
|
||||||
|
|
||||||
GGML_TENSOR_BINARY_OP_LOCALS
|
GGML_TENSOR_BINARY_OP_LOCALS
|
||||||
|
|
||||||
const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
|
const dim3 block_dims(CUDA_GET_ROWS_BLOCK_SIZE, 1, 1);
|
||||||
const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;
|
const int block_num_x = (ne00 + CUDA_GET_ROWS_BLOCK_SIZE - 1) / CUDA_GET_ROWS_BLOCK_SIZE;
|
||||||
const dim3 block_nums(block_num_x, ne10, ne11*ne12);
|
const dim3 block_nums(block_num_x, ne10, ne11*ne12);
|
||||||
|
|
||||||
// strides in elements
|
// strides in elements
|
||||||
//const size_t s0 = nb0 / ggml_element_size(dst);
|
//const size_t s0 = nb0 / ggml_element_size(dst);
|
||||||
const size_t s1 = nb1 / ggml_element_size(dst);
|
const size_t s1 = nb1 / ggml_element_size(dst);
|
||||||
const size_t s2 = nb2 / ggml_element_size(dst);
|
const size_t s2 = nb2 / ggml_element_size(dst);
|
||||||
const size_t s3 = nb3 / ggml_element_size(dst);
|
const size_t s3 = nb3 / ggml_element_size(dst);
|
||||||
|
|
||||||
const size_t s10 = nb10 / ggml_element_size(src1);
|
const size_t s10 = nb10 / ggml_element_size(src1);
|
||||||
const size_t s11 = nb11 / ggml_element_size(src1);
|
const size_t s11 = nb11 / ggml_element_size(src1);
|
||||||
const size_t s12 = nb12 / ggml_element_size(src1);
|
const size_t s12 = nb12 / ggml_element_size(src1);
|
||||||
//const size_t s13 = nb13 / ggml_element_size(src1);
|
//const size_t s13 = nb13 / ggml_element_size(src1);
|
||||||
|
|
||||||
k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(
|
k_get_rows_float<<<block_nums, block_dims, 0, stream>>>(
|
||||||
src0_dd, src1_dd, dst_dd,
|
src0_dd, src1_dd, dst_dd,
|
||||||
ne00, /*ne01, ne02, ne03,*/
|
ne00, /*ne01, ne02, ne03,*/
|
||||||
/*ne10, ne11,*/ ne12, /*ne13,*/
|
/*ne10, ne11,*/ ne12, /*ne13,*/
|
||||||
/* s0,*/ s1, s2, s3,
|
/* s0,*/ s1, s2, s3,
|
||||||
/* nb00,*/ nb01, nb02, nb03,
|
/* nb00,*/ nb01, nb02, nb03,
|
||||||
s10, s11, s12/*, s13*/);
|
s10, s11, s12/*, s13*/);
|
||||||
|
|
||||||
GGML_UNUSED(dst);
|
GGML_UNUSED(dst);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const ggml_tensor * src1 = dst->src[1];
|
const ggml_tensor * src1 = dst->src[1];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
const float * src1_d = (const float *)src1->data;
|
const float * src1_d = (const float *)src1->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
|
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
||||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
|
GGML_ASSERT(src0->nb[0] == ggml_type_size(src0->type));
|
||||||
GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));
|
GGML_ASSERT(src1->nb[0] == ggml_type_size(src1->type));
|
||||||
GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));
|
GGML_ASSERT(dst->nb[0] == ggml_type_size(dst->type));
|
||||||
|
|
||||||
const int32_t * src1_i32 = (const int32_t *) src1_d;
|
const int32_t * src1_i32 = (const int32_t *) src1_d;
|
||||||
|
|
||||||
switch (src0->type) {
|
switch (src0->type) {
|
||||||
case GGML_TYPE_F16:
|
case GGML_TYPE_F16:
|
||||||
get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda_float(src0, src1, dst, (const half *)src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_F32:
|
case GGML_TYPE_F32:
|
||||||
get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda_float(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_Q4_0:
|
case GGML_TYPE_Q4_0:
|
||||||
get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda<QK4_0, QR4_0, dequantize_q4_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_Q4_1:
|
case GGML_TYPE_Q4_1:
|
||||||
get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda<QK4_1, QR4_1, dequantize_q4_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_Q5_0:
|
case GGML_TYPE_Q5_0:
|
||||||
get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda<QK5_0, QR5_0, dequantize_q5_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_Q5_1:
|
case GGML_TYPE_Q5_1:
|
||||||
get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda<QK5_1, QR5_1, dequantize_q5_1>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
case GGML_TYPE_Q8_0:
|
case GGML_TYPE_Q8_0:
|
||||||
get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
get_rows_cuda<QK8_0, QR8_0, dequantize_q8_0>(src0, src1, dst, src0_d, src1_i32, dst_d, stream);
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
// TODO: k-quants
|
// TODO: k-quants
|
||||||
fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));
|
fprintf(stderr, "%s: unsupported type: %s\n", __func__, ggml_type_name(src0->type));
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/getrows.cuh
vendored
10
llama/ggml-cuda/getrows.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_GET_ROWS_BLOCK_SIZE 256
|
#define CUDA_GET_ROWS_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_get_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
208
llama/ggml-cuda/im2col.cu
vendored
208
llama/ggml-cuda/im2col.cu
vendored
@ -1,104 +1,104 @@
|
|||||||
#include "im2col.cuh"
|
#include "im2col.cuh"
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
static __global__ void im2col_kernel(
|
static __global__ void im2col_kernel(
|
||||||
const float * x, T * dst, int64_t batch_offset,
|
const float * x, T * dst, int64_t batch_offset,
|
||||||
int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,
|
int64_t offset_delta, int64_t IC, int64_t IW, int64_t IH, int64_t OH, int64_t OW, int64_t KW, int64_t KH, int64_t pelements, int64_t CHW,
|
||||||
int s0, int s1, int p0, int p1, int d0, int d1) {
|
int s0, int s1, int p0, int p1, int d0, int d1) {
|
||||||
const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;
|
const int64_t i = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
if (i >= pelements) {
|
if (i >= pelements) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int64_t ksize = OW * (KH > 1 ? KW : 1);
|
const int64_t ksize = OW * (KH > 1 ? KW : 1);
|
||||||
const int64_t kx = i / ksize;
|
const int64_t kx = i / ksize;
|
||||||
const int64_t kd = kx * ksize;
|
const int64_t kd = kx * ksize;
|
||||||
const int64_t ky = (i - kd) / OW;
|
const int64_t ky = (i - kd) / OW;
|
||||||
const int64_t ix = i % OW;
|
const int64_t ix = i % OW;
|
||||||
|
|
||||||
const int64_t oh = blockIdx.y;
|
const int64_t oh = blockIdx.y;
|
||||||
const int64_t batch = blockIdx.z / IC;
|
const int64_t batch = blockIdx.z / IC;
|
||||||
const int64_t ic = blockIdx.z % IC;
|
const int64_t ic = blockIdx.z % IC;
|
||||||
|
|
||||||
const int64_t iiw = ix * s0 + kx * d0 - p0;
|
const int64_t iiw = ix * s0 + kx * d0 - p0;
|
||||||
const int64_t iih = oh * s1 + ky * d1 - p1;
|
const int64_t iih = oh * s1 + ky * d1 - p1;
|
||||||
|
|
||||||
const int64_t offset_dst =
|
const int64_t offset_dst =
|
||||||
((batch * OH + oh) * OW + ix) * CHW +
|
((batch * OH + oh) * OW + ix) * CHW +
|
||||||
(ic * (KW * KH) + ky * KW + kx);
|
(ic * (KW * KH) + ky * KW + kx);
|
||||||
|
|
||||||
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
|
if (iih < 0 || iih >= IH || iiw < 0 || iiw >= IW) {
|
||||||
dst[offset_dst] = 0.0f;
|
dst[offset_dst] = 0.0f;
|
||||||
} else {
|
} else {
|
||||||
const int64_t offset_src = ic * offset_delta + batch * batch_offset;
|
const int64_t offset_src = ic * offset_delta + batch * batch_offset;
|
||||||
dst[offset_dst] = x[offset_src + iih * IW + iiw];
|
dst[offset_dst] = x[offset_src + iih * IW + iiw];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
static void im2col_cuda(const float * x, T* dst,
|
static void im2col_cuda(const float * x, T* dst,
|
||||||
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
||||||
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
||||||
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
||||||
const int parallel_elements = OW * KW * KH;
|
const int parallel_elements = OW * KW * KH;
|
||||||
const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
|
const int num_blocks = (parallel_elements + CUDA_IM2COL_BLOCK_SIZE - 1) / CUDA_IM2COL_BLOCK_SIZE;
|
||||||
dim3 block_nums(num_blocks, OH, batch * IC);
|
dim3 block_nums(num_blocks, OH, batch * IC);
|
||||||
im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);
|
im2col_kernel<<<block_nums, CUDA_IM2COL_BLOCK_SIZE, 0, stream>>>(x, dst, batch_offset, offset_delta, IC, IW, IH, OH, OW, KW, KH, parallel_elements, (IC * KH * KW), s0, s1, p0, p1, d0, d1);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void im2col_cuda_f16(const float * x, half * dst,
|
static void im2col_cuda_f16(const float * x, half * dst,
|
||||||
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
||||||
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
||||||
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
||||||
|
|
||||||
im2col_cuda<half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
|
im2col_cuda<half>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void im2col_cuda_f32(const float * x, float * dst,
|
static void im2col_cuda_f32(const float * x, float * dst,
|
||||||
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
int64_t IW, int64_t IH, int64_t OW, int64_t OH, int64_t KW, int64_t KH, int64_t IC,
|
||||||
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
int64_t batch, int64_t batch_offset, int64_t offset_delta,
|
||||||
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
int s0,int s1,int p0,int p1,int d0,int d1, cudaStream_t stream) {
|
||||||
|
|
||||||
im2col_cuda<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
|
im2col_cuda<float>(x, dst, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, offset_delta, s0, s1, p0, p1, d0, d1, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const ggml_tensor * src1 = dst->src[1];
|
const ggml_tensor * src1 = dst->src[1];
|
||||||
const float * src1_d = (const float *)src1->data;
|
const float * src1_d = (const float *)src1->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
GGML_ASSERT(src0->type == GGML_TYPE_F16);
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F16 || dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
|
const int32_t s0 = ((const int32_t*)(dst->op_params))[0];
|
||||||
const int32_t s1 = ((const int32_t*)(dst->op_params))[1];
|
const int32_t s1 = ((const int32_t*)(dst->op_params))[1];
|
||||||
const int32_t p0 = ((const int32_t*)(dst->op_params))[2];
|
const int32_t p0 = ((const int32_t*)(dst->op_params))[2];
|
||||||
const int32_t p1 = ((const int32_t*)(dst->op_params))[3];
|
const int32_t p1 = ((const int32_t*)(dst->op_params))[3];
|
||||||
const int32_t d0 = ((const int32_t*)(dst->op_params))[4];
|
const int32_t d0 = ((const int32_t*)(dst->op_params))[4];
|
||||||
const int32_t d1 = ((const int32_t*)(dst->op_params))[5];
|
const int32_t d1 = ((const int32_t*)(dst->op_params))[5];
|
||||||
|
|
||||||
const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;
|
const bool is_2D = ((const int32_t*)(dst->op_params))[6] == 1;
|
||||||
|
|
||||||
const int64_t IC = src1->ne[is_2D ? 2 : 1];
|
const int64_t IC = src1->ne[is_2D ? 2 : 1];
|
||||||
const int64_t IH = is_2D ? src1->ne[1] : 1;
|
const int64_t IH = is_2D ? src1->ne[1] : 1;
|
||||||
const int64_t IW = src1->ne[0];
|
const int64_t IW = src1->ne[0];
|
||||||
|
|
||||||
const int64_t KH = is_2D ? src0->ne[1] : 1;
|
const int64_t KH = is_2D ? src0->ne[1] : 1;
|
||||||
const int64_t KW = src0->ne[0];
|
const int64_t KW = src0->ne[0];
|
||||||
|
|
||||||
const int64_t OH = is_2D ? dst->ne[2] : 1;
|
const int64_t OH = is_2D ? dst->ne[2] : 1;
|
||||||
const int64_t OW = dst->ne[1];
|
const int64_t OW = dst->ne[1];
|
||||||
|
|
||||||
const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32
|
const size_t delta_offset = src1->nb[is_2D ? 2 : 1] / 4; // nb is byte offset, src is type float32
|
||||||
const int64_t batch = src1->ne[3];
|
const int64_t batch = src1->ne[3];
|
||||||
const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32
|
const size_t batch_offset = src1->nb[3] / 4; // nb is byte offset, src is type float32
|
||||||
|
|
||||||
if(dst->type == GGML_TYPE_F16) {
|
if(dst->type == GGML_TYPE_F16) {
|
||||||
im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
|
im2col_cuda_f16(src1_d, (half *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
|
||||||
} else {
|
} else {
|
||||||
im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
|
im2col_cuda_f32(src1_d, (float *) dst_d, IW, IH, OW, OH, KW, KH, IC, batch, batch_offset, delta_offset, s0, s1, p0, p1, d0, d1, stream);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/im2col.cuh
vendored
10
llama/ggml-cuda/im2col.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_IM2COL_BLOCK_SIZE 256
|
#define CUDA_IM2COL_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_im2col(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
18
llama/ggml-cuda/mmq.cuh
vendored
18
llama/ggml-cuda/mmq.cuh
vendored
@ -1,9 +1,9 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_mul_mat_q(
|
void ggml_cuda_op_mul_mat_q(
|
||||||
ggml_backend_cuda_context & ctx,
|
ggml_backend_cuda_context & ctx,
|
||||||
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
||||||
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
||||||
const int64_t src1_padded_row_size, cudaStream_t stream);
|
const int64_t src1_padded_row_size, cudaStream_t stream);
|
||||||
|
|
||||||
bool ggml_cuda_supports_mmq(enum ggml_type type);
|
bool ggml_cuda_supports_mmq(enum ggml_type type);
|
||||||
|
|||||||
14
llama/ggml-cuda/mmvq.cuh
vendored
14
llama/ggml-cuda/mmvq.cuh
vendored
@ -1,7 +1,7 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_mul_mat_vec_q(
|
void ggml_cuda_op_mul_mat_vec_q(
|
||||||
ggml_backend_cuda_context & ctx,
|
ggml_backend_cuda_context & ctx,
|
||||||
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst, const char * src0_dd_i, const float * src1_ddf_i,
|
||||||
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
const char * src1_ddq_i, float * dst_dd_i, const int64_t row_low, const int64_t row_high, const int64_t src1_ncols,
|
||||||
const int64_t src1_padded_row_size, cudaStream_t stream);
|
const int64_t src1_padded_row_size, cudaStream_t stream);
|
||||||
|
|||||||
430
llama/ggml-cuda/norm.cu
vendored
430
llama/ggml-cuda/norm.cu
vendored
@ -1,215 +1,215 @@
|
|||||||
#include "norm.cuh"
|
#include "norm.cuh"
|
||||||
|
|
||||||
template <int block_size>
|
template <int block_size>
|
||||||
static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {
|
static __global__ void norm_f32(const float * x, float * dst, const int ncols, const float eps) {
|
||||||
const int row = blockIdx.x*blockDim.y + threadIdx.y;
|
const int row = blockIdx.x*blockDim.y + threadIdx.y;
|
||||||
const int tid = threadIdx.x;
|
const int tid = threadIdx.x;
|
||||||
|
|
||||||
float2 mean_var = make_float2(0.f, 0.f);
|
float2 mean_var = make_float2(0.f, 0.f);
|
||||||
|
|
||||||
for (int col = tid; col < ncols; col += block_size) {
|
for (int col = tid; col < ncols; col += block_size) {
|
||||||
const float xi = x[row*ncols + col];
|
const float xi = x[row*ncols + col];
|
||||||
mean_var.x += xi;
|
mean_var.x += xi;
|
||||||
mean_var.y += xi * xi;
|
mean_var.y += xi * xi;
|
||||||
}
|
}
|
||||||
|
|
||||||
// sum up partial sums
|
// sum up partial sums
|
||||||
mean_var = warp_reduce_sum(mean_var);
|
mean_var = warp_reduce_sum(mean_var);
|
||||||
if (block_size > WARP_SIZE) {
|
if (block_size > WARP_SIZE) {
|
||||||
__shared__ float2 s_sum[32];
|
__shared__ float2 s_sum[32];
|
||||||
int warp_id = threadIdx.x / WARP_SIZE;
|
int warp_id = threadIdx.x / WARP_SIZE;
|
||||||
int lane_id = threadIdx.x % WARP_SIZE;
|
int lane_id = threadIdx.x % WARP_SIZE;
|
||||||
if (lane_id == 0) {
|
if (lane_id == 0) {
|
||||||
s_sum[warp_id] = mean_var;
|
s_sum[warp_id] = mean_var;
|
||||||
}
|
}
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
mean_var = s_sum[lane_id];
|
mean_var = s_sum[lane_id];
|
||||||
mean_var = warp_reduce_sum(mean_var);
|
mean_var = warp_reduce_sum(mean_var);
|
||||||
}
|
}
|
||||||
|
|
||||||
const float mean = mean_var.x / ncols;
|
const float mean = mean_var.x / ncols;
|
||||||
const float var = mean_var.y / ncols - mean * mean;
|
const float var = mean_var.y / ncols - mean * mean;
|
||||||
const float inv_std = rsqrtf(var + eps);
|
const float inv_std = rsqrtf(var + eps);
|
||||||
|
|
||||||
for (int col = tid; col < ncols; col += block_size) {
|
for (int col = tid; col < ncols; col += block_size) {
|
||||||
dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;
|
dst[row*ncols + col] = (x[row*ncols + col] - mean) * inv_std;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <int block_size>
|
template <int block_size>
|
||||||
static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {
|
static __global__ void group_norm_f32(const float * x, float * dst, const int group_size, const int ne_elements, const float eps) {
|
||||||
// blockIdx.x: num_groups idx
|
// blockIdx.x: num_groups idx
|
||||||
// threadIdx.x: block_size idx
|
// threadIdx.x: block_size idx
|
||||||
int start = blockIdx.x * group_size;
|
int start = blockIdx.x * group_size;
|
||||||
int end = start + group_size;
|
int end = start + group_size;
|
||||||
|
|
||||||
start += threadIdx.x;
|
start += threadIdx.x;
|
||||||
|
|
||||||
if (end >= ne_elements) {
|
if (end >= ne_elements) {
|
||||||
end = ne_elements;
|
end = ne_elements;
|
||||||
}
|
}
|
||||||
|
|
||||||
float tmp = 0.0f; // partial sum for thread in warp
|
float tmp = 0.0f; // partial sum for thread in warp
|
||||||
|
|
||||||
for (int j = start; j < end; j += block_size) {
|
for (int j = start; j < end; j += block_size) {
|
||||||
tmp += x[j];
|
tmp += x[j];
|
||||||
}
|
}
|
||||||
|
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
if (block_size > WARP_SIZE) {
|
if (block_size > WARP_SIZE) {
|
||||||
__shared__ float s_sum[32];
|
__shared__ float s_sum[32];
|
||||||
int warp_id = threadIdx.x / WARP_SIZE;
|
int warp_id = threadIdx.x / WARP_SIZE;
|
||||||
int lane_id = threadIdx.x % WARP_SIZE;
|
int lane_id = threadIdx.x % WARP_SIZE;
|
||||||
if (lane_id == 0) {
|
if (lane_id == 0) {
|
||||||
s_sum[warp_id] = tmp;
|
s_sum[warp_id] = tmp;
|
||||||
}
|
}
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
tmp = s_sum[lane_id];
|
tmp = s_sum[lane_id];
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
}
|
}
|
||||||
|
|
||||||
float mean = tmp / group_size;
|
float mean = tmp / group_size;
|
||||||
tmp = 0.0f;
|
tmp = 0.0f;
|
||||||
|
|
||||||
for (int j = start; j < end; j += block_size) {
|
for (int j = start; j < end; j += block_size) {
|
||||||
float xi = x[j] - mean;
|
float xi = x[j] - mean;
|
||||||
dst[j] = xi;
|
dst[j] = xi;
|
||||||
tmp += xi * xi;
|
tmp += xi * xi;
|
||||||
}
|
}
|
||||||
|
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
if (block_size > WARP_SIZE) {
|
if (block_size > WARP_SIZE) {
|
||||||
__shared__ float s_sum[32];
|
__shared__ float s_sum[32];
|
||||||
int warp_id = threadIdx.x / WARP_SIZE;
|
int warp_id = threadIdx.x / WARP_SIZE;
|
||||||
int lane_id = threadIdx.x % WARP_SIZE;
|
int lane_id = threadIdx.x % WARP_SIZE;
|
||||||
if (lane_id == 0) {
|
if (lane_id == 0) {
|
||||||
s_sum[warp_id] = tmp;
|
s_sum[warp_id] = tmp;
|
||||||
}
|
}
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
tmp = s_sum[lane_id];
|
tmp = s_sum[lane_id];
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
}
|
}
|
||||||
|
|
||||||
float variance = tmp / group_size;
|
float variance = tmp / group_size;
|
||||||
float scale = rsqrtf(variance + eps);
|
float scale = rsqrtf(variance + eps);
|
||||||
for (int j = start; j < end; j += block_size) {
|
for (int j = start; j < end; j += block_size) {
|
||||||
dst[j] *= scale;
|
dst[j] *= scale;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template <int block_size>
|
template <int block_size>
|
||||||
static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {
|
static __global__ void rms_norm_f32(const float * x, float * dst, const int ncols, const float eps) {
|
||||||
const int row = blockIdx.x*blockDim.y + threadIdx.y;
|
const int row = blockIdx.x*blockDim.y + threadIdx.y;
|
||||||
const int tid = threadIdx.x;
|
const int tid = threadIdx.x;
|
||||||
|
|
||||||
float tmp = 0.0f; // partial sum for thread in warp
|
float tmp = 0.0f; // partial sum for thread in warp
|
||||||
|
|
||||||
for (int col = tid; col < ncols; col += block_size) {
|
for (int col = tid; col < ncols; col += block_size) {
|
||||||
const float xi = x[row*ncols + col];
|
const float xi = x[row*ncols + col];
|
||||||
tmp += xi * xi;
|
tmp += xi * xi;
|
||||||
}
|
}
|
||||||
|
|
||||||
// sum up partial sums
|
// sum up partial sums
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
if (block_size > WARP_SIZE) {
|
if (block_size > WARP_SIZE) {
|
||||||
__shared__ float s_sum[32];
|
__shared__ float s_sum[32];
|
||||||
int warp_id = threadIdx.x / WARP_SIZE;
|
int warp_id = threadIdx.x / WARP_SIZE;
|
||||||
int lane_id = threadIdx.x % WARP_SIZE;
|
int lane_id = threadIdx.x % WARP_SIZE;
|
||||||
if (lane_id == 0) {
|
if (lane_id == 0) {
|
||||||
s_sum[warp_id] = tmp;
|
s_sum[warp_id] = tmp;
|
||||||
}
|
}
|
||||||
__syncthreads();
|
__syncthreads();
|
||||||
tmp = s_sum[lane_id];
|
tmp = s_sum[lane_id];
|
||||||
tmp = warp_reduce_sum(tmp);
|
tmp = warp_reduce_sum(tmp);
|
||||||
}
|
}
|
||||||
|
|
||||||
const float mean = tmp / ncols;
|
const float mean = tmp / ncols;
|
||||||
const float scale = rsqrtf(mean + eps);
|
const float scale = rsqrtf(mean + eps);
|
||||||
|
|
||||||
for (int col = tid; col < ncols; col += block_size) {
|
for (int col = tid; col < ncols; col += block_size) {
|
||||||
dst[row*ncols + col] = scale * x[row*ncols + col];
|
dst[row*ncols + col] = scale * x[row*ncols + col];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
|
static void norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
|
||||||
GGML_ASSERT(ncols % WARP_SIZE == 0);
|
GGML_ASSERT(ncols % WARP_SIZE == 0);
|
||||||
if (ncols < 1024) {
|
if (ncols < 1024) {
|
||||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||||
norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
||||||
} else {
|
} else {
|
||||||
const dim3 block_dims(1024, 1, 1);
|
const dim3 block_dims(1024, 1, 1);
|
||||||
norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {
|
static void group_norm_f32_cuda(const float * x, float * dst, const int num_groups, const int group_size, const int ne_elements, cudaStream_t stream) {
|
||||||
static const float eps = 1e-6f;
|
static const float eps = 1e-6f;
|
||||||
if (group_size < 1024) {
|
if (group_size < 1024) {
|
||||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||||
group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
group_norm_f32<WARP_SIZE><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
||||||
} else {
|
} else {
|
||||||
const dim3 block_dims(1024, 1, 1);
|
const dim3 block_dims(1024, 1, 1);
|
||||||
group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
group_norm_f32<1024><<<num_groups, block_dims, 0, stream>>>(x, dst, group_size, ne_elements, eps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
|
static void rms_norm_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, const float eps, cudaStream_t stream) {
|
||||||
GGML_ASSERT(ncols % WARP_SIZE == 0);
|
GGML_ASSERT(ncols % WARP_SIZE == 0);
|
||||||
if (ncols < 1024) {
|
if (ncols < 1024) {
|
||||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||||
rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
rms_norm_f32<WARP_SIZE><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
||||||
} else {
|
} else {
|
||||||
const dim3 block_dims(1024, 1, 1);
|
const dim3 block_dims(1024, 1, 1);
|
||||||
rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
rms_norm_f32<1024><<<nrows, block_dims, 0, stream>>>(x, dst, ncols, eps);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int64_t ne00 = src0->ne[0];
|
const int64_t ne00 = src0->ne[0];
|
||||||
const int64_t nrows = ggml_nrows(src0);
|
const int64_t nrows = ggml_nrows(src0);
|
||||||
|
|
||||||
float eps;
|
float eps;
|
||||||
memcpy(&eps, dst->op_params, sizeof(float));
|
memcpy(&eps, dst->op_params, sizeof(float));
|
||||||
|
|
||||||
norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
|
norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
int num_groups = dst->op_params[0];
|
int num_groups = dst->op_params[0];
|
||||||
int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);
|
int group_size = src0->ne[0] * src0->ne[1] * ((src0->ne[2] + num_groups - 1) / num_groups);
|
||||||
group_norm_f32_cuda(src0_d, dst_d, num_groups * src0->ne[3], group_size, ggml_nelements(src0), stream);
|
group_norm_f32_cuda(src0_d, dst_d, num_groups * src0->ne[3], group_size, ggml_nelements(src0), stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int64_t ne00 = src0->ne[0];
|
const int64_t ne00 = src0->ne[0];
|
||||||
const int64_t nrows = ggml_nrows(src0);
|
const int64_t nrows = ggml_nrows(src0);
|
||||||
|
|
||||||
float eps;
|
float eps;
|
||||||
memcpy(&eps, dst->op_params, sizeof(float));
|
memcpy(&eps, dst->op_params, sizeof(float));
|
||||||
|
|
||||||
rms_norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
|
rms_norm_f32_cuda(src0_d, dst_d, ne00, nrows, eps, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
14
llama/ggml-cuda/norm.cuh
vendored
14
llama/ggml-cuda/norm.cuh
vendored
@ -1,7 +1,7 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|
||||||
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_group_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|
||||||
void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_rms_norm(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
98
llama/ggml-cuda/pad.cu
vendored
98
llama/ggml-cuda/pad.cu
vendored
@ -1,49 +1,49 @@
|
|||||||
#include "pad.cuh"
|
#include "pad.cuh"
|
||||||
|
|
||||||
static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) {
|
static __global__ void pad_f32(const float * x, float * dst, const int ne0, const int ne00, const int ne01, const int ne02, const int ne03) {
|
||||||
// blockIdx.z: idx of ne2*ne3, aka ne02*ne03
|
// blockIdx.z: idx of ne2*ne3, aka ne02*ne03
|
||||||
// blockIdx.y: idx of ne1
|
// blockIdx.y: idx of ne1
|
||||||
// blockIDx.x: idx of ne0 / BLOCK_SIZE
|
// blockIDx.x: idx of ne0 / BLOCK_SIZE
|
||||||
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
int nidx = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
if (nidx >= ne0) {
|
if (nidx >= ne0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// operation
|
// operation
|
||||||
int offset_dst =
|
int offset_dst =
|
||||||
nidx +
|
nidx +
|
||||||
blockIdx.y * ne0 +
|
blockIdx.y * ne0 +
|
||||||
blockIdx.z * ne0 * gridDim.y;
|
blockIdx.z * ne0 * gridDim.y;
|
||||||
if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02*ne03) {
|
if (nidx < ne00 && blockIdx.y < ne01 && blockIdx.z < ne02*ne03) {
|
||||||
int offset_src =
|
int offset_src =
|
||||||
nidx +
|
nidx +
|
||||||
blockIdx.y * ne00 +
|
blockIdx.y * ne00 +
|
||||||
blockIdx.z * ne00 * ne01;
|
blockIdx.z * ne00 * ne01;
|
||||||
dst[offset_dst] = x[offset_src];
|
dst[offset_dst] = x[offset_src];
|
||||||
} else {
|
} else {
|
||||||
dst[offset_dst] = 0.0f;
|
dst[offset_dst] = 0.0f;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void pad_f32_cuda(const float * x, float * dst,
|
static void pad_f32_cuda(const float * x, float * dst,
|
||||||
const int ne00, const int ne01, const int ne02, const int ne03,
|
const int ne00, const int ne01, const int ne02, const int ne03,
|
||||||
const int ne0, const int ne1, const int ne2, const int ne3, cudaStream_t stream) {
|
const int ne0, const int ne1, const int ne2, const int ne3, cudaStream_t stream) {
|
||||||
int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;
|
int num_blocks = (ne0 + CUDA_PAD_BLOCK_SIZE - 1) / CUDA_PAD_BLOCK_SIZE;
|
||||||
dim3 gridDim(num_blocks, ne1, ne2*ne3);
|
dim3 gridDim(num_blocks, ne1, ne2*ne3);
|
||||||
pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(x, dst, ne0, ne00, ne01, ne02, ne03);
|
pad_f32<<<gridDim, CUDA_PAD_BLOCK_SIZE, 0, stream>>>(x, dst, ne0, ne00, ne01, ne02, ne03);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors
|
GGML_ASSERT(src0->ne[3] == 1 && dst->ne[3] == 1); // just 3D tensors
|
||||||
|
|
||||||
pad_f32_cuda(src0_d, dst_d,
|
pad_f32_cuda(src0_d, dst_d,
|
||||||
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
|
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3],
|
||||||
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], stream);
|
dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/pad.cuh
vendored
10
llama/ggml-cuda/pad.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_PAD_BLOCK_SIZE 256
|
#define CUDA_PAD_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_pad(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
188
llama/ggml-cuda/pool2d.cu
vendored
188
llama/ggml-cuda/pool2d.cu
vendored
@ -1,94 +1,94 @@
|
|||||||
#include "pool2d.cuh"
|
#include "pool2d.cuh"
|
||||||
|
|
||||||
template <typename Ti, typename To>
|
template <typename Ti, typename To>
|
||||||
static __global__ void pool2d_nchw_kernel(
|
static __global__ void pool2d_nchw_kernel(
|
||||||
const int ih, const int iw, const int oh, const int ow,
|
const int ih, const int iw, const int oh, const int ow,
|
||||||
const int kh, const int kw, const int sh, const int sw,
|
const int kh, const int kw, const int sh, const int sw,
|
||||||
const int ph, const int pw, const int parallel_elements,
|
const int ph, const int pw, const int parallel_elements,
|
||||||
const Ti* src, To* dst, const enum ggml_op_pool op) {
|
const Ti* src, To* dst, const enum ggml_op_pool op) {
|
||||||
int idx = threadIdx.x + blockIdx.x * blockDim.x;
|
int idx = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
if (idx >= parallel_elements) {
|
if (idx >= parallel_elements) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int I_HW = ih * iw;
|
const int I_HW = ih * iw;
|
||||||
const int O_HW = oh * ow;
|
const int O_HW = oh * ow;
|
||||||
const int nc = idx / O_HW;
|
const int nc = idx / O_HW;
|
||||||
const int cur_oh = idx % O_HW / ow;
|
const int cur_oh = idx % O_HW / ow;
|
||||||
const int cur_ow = idx % O_HW % ow;
|
const int cur_ow = idx % O_HW % ow;
|
||||||
const Ti* i_ptr = src + nc * I_HW;
|
const Ti* i_ptr = src + nc * I_HW;
|
||||||
To* o_ptr = dst + nc * O_HW;
|
To* o_ptr = dst + nc * O_HW;
|
||||||
const int start_h = cur_oh * sh - ph;
|
const int start_h = cur_oh * sh - ph;
|
||||||
const int bh = max(0, start_h);
|
const int bh = max(0, start_h);
|
||||||
const int eh = min(ih, start_h + kh);
|
const int eh = min(ih, start_h + kh);
|
||||||
const int start_w = cur_ow * sw - pw;
|
const int start_w = cur_ow * sw - pw;
|
||||||
const int bw = max(0, start_w);
|
const int bw = max(0, start_w);
|
||||||
const int ew = min(iw, start_w + kw);
|
const int ew = min(iw, start_w + kw);
|
||||||
const To scale = 1. / (kh * kw);
|
const To scale = 1. / (kh * kw);
|
||||||
To res = 0;
|
To res = 0;
|
||||||
|
|
||||||
switch (op) {
|
switch (op) {
|
||||||
case GGML_OP_POOL_AVG: res = 0; break;
|
case GGML_OP_POOL_AVG: res = 0; break;
|
||||||
case GGML_OP_POOL_MAX: res = -FLT_MAX; break;
|
case GGML_OP_POOL_MAX: res = -FLT_MAX; break;
|
||||||
default: assert(false);
|
default: assert(false);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (int i = bh; i < eh; i += 1) {
|
for (int i = bh; i < eh; i += 1) {
|
||||||
for (int j = bw; j < ew; j += 1) {
|
for (int j = bw; j < ew; j += 1) {
|
||||||
#if __CUDA_ARCH__ >= 350
|
#if __CUDA_ARCH__ >= 350
|
||||||
Ti cur = __ldg(i_ptr + i * iw + j);
|
Ti cur = __ldg(i_ptr + i * iw + j);
|
||||||
#else
|
#else
|
||||||
Ti cur = i_ptr[i * iw + j];
|
Ti cur = i_ptr[i * iw + j];
|
||||||
#endif
|
#endif
|
||||||
switch (op) {
|
switch (op) {
|
||||||
case GGML_OP_POOL_AVG: res += cur * scale; break;
|
case GGML_OP_POOL_AVG: res += cur * scale; break;
|
||||||
case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;
|
case GGML_OP_POOL_MAX: res = max(res, (To)cur); break;
|
||||||
default: assert(false);
|
default: assert(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
o_ptr[cur_oh * ow + cur_ow] = res;
|
o_ptr[cur_oh * ow + cur_ow] = res;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void pool2d_nchw_kernel_f32_f32_cuda(
|
static void pool2d_nchw_kernel_f32_f32_cuda(
|
||||||
const int ih, const int iw, const int oh, const int ow,
|
const int ih, const int iw, const int oh, const int ow,
|
||||||
const int kh, const int kw, const int sh, const int sw,
|
const int kh, const int kw, const int sh, const int sw,
|
||||||
const int ph, const int pw, const int parallel_elements,
|
const int ph, const int pw, const int parallel_elements,
|
||||||
const float * src, float * dst, const enum ggml_op_pool op,
|
const float * src, float * dst, const enum ggml_op_pool op,
|
||||||
cudaStream_t stream) {
|
cudaStream_t stream) {
|
||||||
|
|
||||||
const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;
|
const int num_blocks = (parallel_elements + CUDA_POOL2D_BLOCK_SIZE - 1) / CUDA_POOL2D_BLOCK_SIZE;
|
||||||
dim3 block_nums(num_blocks);
|
dim3 block_nums(num_blocks);
|
||||||
pool2d_nchw_kernel<<<block_nums, CUDA_POOL2D_BLOCK_SIZE, 0, stream>>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op);
|
pool2d_nchw_kernel<<<block_nums, CUDA_POOL2D_BLOCK_SIZE, 0, stream>>>(ih, iw, oh, ow, kh, kw, sh, sw, ph, pw, parallel_elements, src, dst, op);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int32_t * opts = (const int32_t *)dst->op_params;
|
const int32_t * opts = (const int32_t *)dst->op_params;
|
||||||
enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
|
enum ggml_op_pool op = static_cast<ggml_op_pool>(opts[0]);
|
||||||
const int k0 = opts[1];
|
const int k0 = opts[1];
|
||||||
const int k1 = opts[2];
|
const int k1 = opts[2];
|
||||||
const int s0 = opts[3];
|
const int s0 = opts[3];
|
||||||
const int s1 = opts[4];
|
const int s1 = opts[4];
|
||||||
const int p0 = opts[5];
|
const int p0 = opts[5];
|
||||||
const int p1 = opts[6];
|
const int p1 = opts[6];
|
||||||
|
|
||||||
const int64_t IH = src0->ne[1];
|
const int64_t IH = src0->ne[1];
|
||||||
const int64_t IW = src0->ne[0];
|
const int64_t IW = src0->ne[0];
|
||||||
|
|
||||||
const int64_t N = dst->ne[3];
|
const int64_t N = dst->ne[3];
|
||||||
const int64_t OC = dst->ne[2];
|
const int64_t OC = dst->ne[2];
|
||||||
const int64_t OH = dst->ne[1];
|
const int64_t OH = dst->ne[1];
|
||||||
const int64_t OW = dst->ne[0];
|
const int64_t OW = dst->ne[0];
|
||||||
|
|
||||||
const int parallel_elements = N * OC * OH * OW;
|
const int parallel_elements = N * OC * OH * OW;
|
||||||
|
|
||||||
pool2d_nchw_kernel_f32_f32_cuda(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_d, dst_d, op, stream);
|
pool2d_nchw_kernel_f32_f32_cuda(IH, IW, OH, OW, k1, k0, s1, s0, p1, p0, parallel_elements, src0_d, dst_d, op, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/pool2d.cuh
vendored
10
llama/ggml-cuda/pool2d.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_POOL2D_BLOCK_SIZE 256
|
#define CUDA_POOL2D_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_pool2d(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
90
llama/ggml-cuda/quantize.cu
vendored
90
llama/ggml-cuda/quantize.cu
vendored
@ -1,45 +1,45 @@
|
|||||||
#include "quantize.cuh"
|
#include "quantize.cuh"
|
||||||
|
|
||||||
static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int64_t kx, const int64_t kx_padded) {
|
static __global__ void quantize_q8_1(const float * __restrict__ x, void * __restrict__ vy, const int64_t kx, const int64_t kx_padded) {
|
||||||
const int64_t ix = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
|
const int64_t ix = (int64_t)blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
|
|
||||||
if (ix >= kx_padded) {
|
if (ix >= kx_padded) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int64_t iy = (int64_t)blockDim.y*blockIdx.y + threadIdx.y;
|
const int64_t iy = (int64_t)blockDim.y*blockIdx.y + threadIdx.y;
|
||||||
|
|
||||||
const int64_t i_padded = (int64_t)iy*kx_padded + ix;
|
const int64_t i_padded = (int64_t)iy*kx_padded + ix;
|
||||||
|
|
||||||
block_q8_1 * y = (block_q8_1 *) vy;
|
block_q8_1 * y = (block_q8_1 *) vy;
|
||||||
|
|
||||||
const int64_t ib = i_padded / QK8_1; // block index
|
const int64_t ib = i_padded / QK8_1; // block index
|
||||||
const int64_t iqs = i_padded % QK8_1; // quant index
|
const int64_t iqs = i_padded % QK8_1; // quant index
|
||||||
|
|
||||||
const float xi = ix < kx ? x[iy*kx + ix] : 0.0f;
|
const float xi = ix < kx ? x[iy*kx + ix] : 0.0f;
|
||||||
float amax = fabsf(xi);
|
float amax = fabsf(xi);
|
||||||
float sum = xi;
|
float sum = xi;
|
||||||
|
|
||||||
amax = warp_reduce_max(amax);
|
amax = warp_reduce_max(amax);
|
||||||
sum = warp_reduce_sum(sum);
|
sum = warp_reduce_sum(sum);
|
||||||
|
|
||||||
const float d = amax / 127;
|
const float d = amax / 127;
|
||||||
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
|
const int8_t q = amax == 0.0f ? 0 : roundf(xi / d);
|
||||||
|
|
||||||
y[ib].qs[iqs] = q;
|
y[ib].qs[iqs] = q;
|
||||||
|
|
||||||
if (iqs > 0) {
|
if (iqs > 0) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
reinterpret_cast<half&>(y[ib].ds.x) = d;
|
reinterpret_cast<half&>(y[ib].ds.x) = d;
|
||||||
reinterpret_cast<half&>(y[ib].ds.y) = sum;
|
reinterpret_cast<half&>(y[ib].ds.y) = sum;
|
||||||
}
|
}
|
||||||
|
|
||||||
void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream) {
|
void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream) {
|
||||||
const int64_t block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
|
const int64_t block_num_x = (kx_padded + CUDA_QUANTIZE_BLOCK_SIZE - 1) / CUDA_QUANTIZE_BLOCK_SIZE;
|
||||||
const dim3 num_blocks(block_num_x, ky, 1);
|
const dim3 num_blocks(block_num_x, ky, 1);
|
||||||
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
|
const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE, 1, 1);
|
||||||
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);
|
quantize_q8_1<<<num_blocks, block_size, 0, stream>>>(x, vy, kx, kx_padded);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
10
llama/ggml-cuda/quantize.cuh
vendored
10
llama/ggml-cuda/quantize.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_QUANTIZE_BLOCK_SIZE 256
|
#define CUDA_QUANTIZE_BLOCK_SIZE 256
|
||||||
|
|
||||||
void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream);
|
void quantize_row_q8_1_cuda(const float * x, void * vy, const int64_t kx, const int64_t ky, const int64_t kx_padded, cudaStream_t stream);
|
||||||
|
|||||||
616
llama/ggml-cuda/rope.cu
vendored
616
llama/ggml-cuda/rope.cu
vendored
@ -1,308 +1,308 @@
|
|||||||
#include "rope.cuh"
|
#include "rope.cuh"
|
||||||
|
|
||||||
struct rope_corr_dims {
|
struct rope_corr_dims {
|
||||||
float v[4];
|
float v[4];
|
||||||
};
|
};
|
||||||
|
|
||||||
static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {
|
static __device__ float rope_yarn_ramp(const float low, const float high, const int i0) {
|
||||||
const float y = (i0 / 2 - low) / max(0.001f, high - low);
|
const float y = (i0 / 2 - low) / max(0.001f, high - low);
|
||||||
return 1.0f - min(1.0f, max(0.0f, y));
|
return 1.0f - min(1.0f, max(0.0f, y));
|
||||||
}
|
}
|
||||||
|
|
||||||
// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
|
// YaRN algorithm based on LlamaYaRNScaledRotaryEmbedding.py from https://github.com/jquesnelle/yarn
|
||||||
// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
|
// MIT licensed. Copyright (c) 2023 Jeffrey Quesnelle and Bowen Peng.
|
||||||
static __device__ void rope_yarn(
|
static __device__ void rope_yarn(
|
||||||
float theta_extrap, float freq_scale, rope_corr_dims corr_dims, int64_t i0, float ext_factor, float mscale,
|
float theta_extrap, float freq_scale, rope_corr_dims corr_dims, int64_t i0, float ext_factor, float mscale,
|
||||||
float * cos_theta, float * sin_theta
|
float * cos_theta, float * sin_theta
|
||||||
) {
|
) {
|
||||||
// Get n-d rotational scaling corrected for extrapolation
|
// Get n-d rotational scaling corrected for extrapolation
|
||||||
float theta_interp = freq_scale * theta_extrap;
|
float theta_interp = freq_scale * theta_extrap;
|
||||||
float theta = theta_interp;
|
float theta = theta_interp;
|
||||||
if (ext_factor != 0.0f) {
|
if (ext_factor != 0.0f) {
|
||||||
float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;
|
float ramp_mix = rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;
|
||||||
theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
|
theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
|
||||||
|
|
||||||
// Get n-d magnitude scaling corrected for interpolation
|
// Get n-d magnitude scaling corrected for interpolation
|
||||||
mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
|
mscale *= 1.0f + 0.1f * logf(1.0f / freq_scale);
|
||||||
}
|
}
|
||||||
*cos_theta = cosf(theta) * mscale;
|
*cos_theta = cosf(theta) * mscale;
|
||||||
*sin_theta = sinf(theta) * mscale;
|
*sin_theta = sinf(theta) * mscale;
|
||||||
}
|
}
|
||||||
|
|
||||||
// rope == RoPE == rotary positional embedding
|
// rope == RoPE == rotary positional embedding
|
||||||
template<typename T, bool has_pos>
|
template<typename T, bool has_pos>
|
||||||
static __global__ void rope(
|
static __global__ void rope(
|
||||||
const T * x, T * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,
|
const T * x, T * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,
|
||||||
float ext_factor, float attn_factor, rope_corr_dims corr_dims
|
float ext_factor, float attn_factor, rope_corr_dims corr_dims
|
||||||
) {
|
) {
|
||||||
const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);
|
const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);
|
||||||
|
|
||||||
if (col >= ncols) {
|
if (col >= ncols) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
const int i = row*ncols + col;
|
const int i = row*ncols + col;
|
||||||
const int i2 = row/p_delta_rows;
|
const int i2 = row/p_delta_rows;
|
||||||
|
|
||||||
const int p = has_pos ? pos[i2] : 0;
|
const int p = has_pos ? pos[i2] : 0;
|
||||||
const float theta_base = p*powf(freq_base, -float(col)/ncols);
|
const float theta_base = p*powf(freq_base, -float(col)/ncols);
|
||||||
|
|
||||||
float cos_theta, sin_theta;
|
float cos_theta, sin_theta;
|
||||||
rope_yarn(theta_base, freq_scale, corr_dims, col, ext_factor, attn_factor, &cos_theta, &sin_theta);
|
rope_yarn(theta_base, freq_scale, corr_dims, col, ext_factor, attn_factor, &cos_theta, &sin_theta);
|
||||||
|
|
||||||
const float x0 = x[i + 0];
|
const float x0 = x[i + 0];
|
||||||
const float x1 = x[i + 1];
|
const float x1 = x[i + 1];
|
||||||
|
|
||||||
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
||||||
dst[i + 1] = x0*sin_theta + x1*cos_theta;
|
dst[i + 1] = x0*sin_theta + x1*cos_theta;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T, bool has_pos>
|
template<typename T, bool has_pos>
|
||||||
static __global__ void rope_neox(
|
static __global__ void rope_neox(
|
||||||
const T * x, T * dst, int ncols, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const T * x, T * dst, int ncols, int n_dims, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims
|
float ext_factor, float attn_factor, rope_corr_dims corr_dims, float theta_scale, float inv_ndims
|
||||||
) {
|
) {
|
||||||
const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);
|
const int col = 2*(blockDim.y*blockIdx.y + threadIdx.y);
|
||||||
|
|
||||||
if (col >= ncols) {
|
if (col >= ncols) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
const int row = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
const int ib = col / n_dims;
|
const int ib = col / n_dims;
|
||||||
const int ic = col % n_dims;
|
const int ic = col % n_dims;
|
||||||
|
|
||||||
if (ib > 0) {
|
if (ib > 0) {
|
||||||
const int i = row*ncols + ib*n_dims + ic;
|
const int i = row*ncols + ib*n_dims + ic;
|
||||||
|
|
||||||
dst[i + 0] = x[i + 0];
|
dst[i + 0] = x[i + 0];
|
||||||
dst[i + 1] = x[i + 1];
|
dst[i + 1] = x[i + 1];
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int i = row*ncols + ib*n_dims + ic/2;
|
const int i = row*ncols + ib*n_dims + ic/2;
|
||||||
const int i2 = row/p_delta_rows;
|
const int i2 = row/p_delta_rows;
|
||||||
|
|
||||||
float cur_rot = inv_ndims * ic - ib;
|
float cur_rot = inv_ndims * ic - ib;
|
||||||
|
|
||||||
const int p = has_pos ? pos[i2] : 0;
|
const int p = has_pos ? pos[i2] : 0;
|
||||||
const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f);
|
const float theta_base = p*freq_scale*powf(theta_scale, col/2.0f);
|
||||||
|
|
||||||
float cos_theta, sin_theta;
|
float cos_theta, sin_theta;
|
||||||
rope_yarn(theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);
|
rope_yarn(theta_base, freq_scale, corr_dims, cur_rot, ext_factor, attn_factor, &cos_theta, &sin_theta);
|
||||||
|
|
||||||
const float x0 = x[i + 0];
|
const float x0 = x[i + 0];
|
||||||
const float x1 = x[i + n_dims/2];
|
const float x1 = x[i + n_dims/2];
|
||||||
|
|
||||||
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
||||||
dst[i + n_dims/2] = x0*sin_theta + x1*cos_theta;
|
dst[i + n_dims/2] = x0*sin_theta + x1*cos_theta;
|
||||||
}
|
}
|
||||||
|
|
||||||
static __global__ void rope_glm_f32(
|
static __global__ void rope_glm_f32(
|
||||||
const float * x, float * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,
|
const float * x, float * dst, int ncols, const int32_t * pos, float freq_scale, int p_delta_rows, float freq_base,
|
||||||
int n_ctx
|
int n_ctx
|
||||||
) {
|
) {
|
||||||
const int col = blockDim.x*blockIdx.x + threadIdx.x;
|
const int col = blockDim.x*blockIdx.x + threadIdx.x;
|
||||||
const int half_n_dims = ncols/4;
|
const int half_n_dims = ncols/4;
|
||||||
|
|
||||||
if (col >= half_n_dims) {
|
if (col >= half_n_dims) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const int row = blockDim.y*blockIdx.y + threadIdx.y;
|
const int row = blockDim.y*blockIdx.y + threadIdx.y;
|
||||||
const int i = row*ncols + col;
|
const int i = row*ncols + col;
|
||||||
const int i2 = row/p_delta_rows;
|
const int i2 = row/p_delta_rows;
|
||||||
|
|
||||||
const float col_theta_scale = powf(freq_base, -2.0f*col/ncols);
|
const float col_theta_scale = powf(freq_base, -2.0f*col/ncols);
|
||||||
// FIXME: this is likely wrong
|
// FIXME: this is likely wrong
|
||||||
const int p = pos != nullptr ? pos[i2] : 0;
|
const int p = pos != nullptr ? pos[i2] : 0;
|
||||||
|
|
||||||
const float theta = min(p, n_ctx - 2)*freq_scale*col_theta_scale;
|
const float theta = min(p, n_ctx - 2)*freq_scale*col_theta_scale;
|
||||||
const float sin_theta = sinf(theta);
|
const float sin_theta = sinf(theta);
|
||||||
const float cos_theta = cosf(theta);
|
const float cos_theta = cosf(theta);
|
||||||
|
|
||||||
const float x0 = x[i + 0];
|
const float x0 = x[i + 0];
|
||||||
const float x1 = x[i + half_n_dims];
|
const float x1 = x[i + half_n_dims];
|
||||||
|
|
||||||
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
dst[i + 0] = x0*cos_theta - x1*sin_theta;
|
||||||
dst[i + half_n_dims] = x0*sin_theta + x1*cos_theta;
|
dst[i + half_n_dims] = x0*sin_theta + x1*cos_theta;
|
||||||
|
|
||||||
const float block_theta = ((float)max(p - n_ctx - 2, 0))*col_theta_scale;
|
const float block_theta = ((float)max(p - n_ctx - 2, 0))*col_theta_scale;
|
||||||
const float sin_block_theta = sinf(block_theta);
|
const float sin_block_theta = sinf(block_theta);
|
||||||
const float cos_block_theta = cosf(block_theta);
|
const float cos_block_theta = cosf(block_theta);
|
||||||
|
|
||||||
const float x2 = x[i + half_n_dims * 2];
|
const float x2 = x[i + half_n_dims * 2];
|
||||||
const float x3 = x[i + half_n_dims * 3];
|
const float x3 = x[i + half_n_dims * 3];
|
||||||
|
|
||||||
dst[i + half_n_dims * 2] = x2*cos_block_theta - x3*sin_block_theta;
|
dst[i + half_n_dims * 2] = x2*cos_block_theta - x3*sin_block_theta;
|
||||||
dst[i + half_n_dims * 3] = x2*sin_block_theta + x3*cos_block_theta;
|
dst[i + half_n_dims * 3] = x2*sin_block_theta + x3*cos_block_theta;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
static void rope_cuda(
|
static void rope_cuda(
|
||||||
const T * x, T * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const T * x, T * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
||||||
) {
|
) {
|
||||||
GGML_ASSERT(ncols % 2 == 0);
|
GGML_ASSERT(ncols % 2 == 0);
|
||||||
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
|
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
|
||||||
const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
|
const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
|
||||||
const dim3 block_nums(nrows, num_blocks_x, 1);
|
const dim3 block_nums(nrows, num_blocks_x, 1);
|
||||||
if (pos == nullptr) {
|
if (pos == nullptr) {
|
||||||
rope<T, false><<<block_nums, block_dims, 0, stream>>>(
|
rope<T, false><<<block_nums, block_dims, 0, stream>>>(
|
||||||
x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims
|
x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
rope<T, true><<<block_nums, block_dims, 0, stream>>>(
|
rope<T, true><<<block_nums, block_dims, 0, stream>>>(
|
||||||
x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims
|
x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename T>
|
template<typename T>
|
||||||
static void rope_neox_cuda(
|
static void rope_neox_cuda(
|
||||||
const T * x, T * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const T * x, T * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
||||||
) {
|
) {
|
||||||
GGML_ASSERT(ncols % 2 == 0);
|
GGML_ASSERT(ncols % 2 == 0);
|
||||||
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
|
const dim3 block_dims(1, CUDA_ROPE_BLOCK_SIZE, 1);
|
||||||
const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
|
const int num_blocks_x = (ncols + 2*CUDA_ROPE_BLOCK_SIZE - 1) / (2*CUDA_ROPE_BLOCK_SIZE);
|
||||||
const dim3 block_nums(nrows, num_blocks_x, 1);
|
const dim3 block_nums(nrows, num_blocks_x, 1);
|
||||||
|
|
||||||
const float theta_scale = powf(freq_base, -2.0f/n_dims);
|
const float theta_scale = powf(freq_base, -2.0f/n_dims);
|
||||||
const float inv_ndims = -1.0f / n_dims;
|
const float inv_ndims = -1.0f / n_dims;
|
||||||
|
|
||||||
if (pos == nullptr) {
|
if (pos == nullptr) {
|
||||||
rope_neox<T, false><<<block_nums, block_dims, 0, stream>>>(
|
rope_neox<T, false><<<block_nums, block_dims, 0, stream>>>(
|
||||||
x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
|
x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
|
||||||
theta_scale, inv_ndims
|
theta_scale, inv_ndims
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
rope_neox<T, true><<<block_nums, block_dims, 0, stream>>>(
|
rope_neox<T, true><<<block_nums, block_dims, 0, stream>>>(
|
||||||
x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
|
x, dst, ncols, n_dims, pos, freq_scale, p_delta_rows, ext_factor, attn_factor, corr_dims,
|
||||||
theta_scale, inv_ndims
|
theta_scale, inv_ndims
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rope_glm_f32_cuda(
|
static void rope_glm_f32_cuda(
|
||||||
const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, int n_ctx, cudaStream_t stream
|
float freq_base, int n_ctx, cudaStream_t stream
|
||||||
) {
|
) {
|
||||||
GGML_ASSERT(ncols % 4 == 0);
|
GGML_ASSERT(ncols % 4 == 0);
|
||||||
const dim3 block_dims(CUDA_ROPE_BLOCK_SIZE/4, 1, 1);
|
const dim3 block_dims(CUDA_ROPE_BLOCK_SIZE/4, 1, 1);
|
||||||
const int num_blocks_x = (ncols + CUDA_ROPE_BLOCK_SIZE - 1) / CUDA_ROPE_BLOCK_SIZE;
|
const int num_blocks_x = (ncols + CUDA_ROPE_BLOCK_SIZE - 1) / CUDA_ROPE_BLOCK_SIZE;
|
||||||
const dim3 block_nums(num_blocks_x, nrows, 1);
|
const dim3 block_nums(num_blocks_x, nrows, 1);
|
||||||
rope_glm_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, n_ctx);
|
rope_glm_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols, pos, freq_scale, p_delta_rows, freq_base, n_ctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rope_cuda_f16(
|
static void rope_cuda_f16(
|
||||||
const half * x, half * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const half * x, half * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
||||||
|
|
||||||
rope_cuda<half>(x, dst, ncols, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
rope_cuda<half>(x, dst, ncols, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rope_cuda_f32(
|
static void rope_cuda_f32(
|
||||||
const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const float * x, float * dst, int ncols, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
||||||
|
|
||||||
rope_cuda<float>(x, dst, ncols, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
rope_cuda<float>(x, dst, ncols, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rope_neox_cuda_f16(
|
static void rope_neox_cuda_f16(
|
||||||
const half * x, half * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const half * x, half * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream) {
|
||||||
|
|
||||||
rope_neox_cuda<half>(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
rope_neox_cuda<half>(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void rope_neox_cuda_f32(
|
static void rope_neox_cuda_f32(
|
||||||
const float * x, float * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
const float * x, float * dst, int ncols, int n_dims, int nrows, const int32_t * pos, float freq_scale, int p_delta_rows,
|
||||||
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
float freq_base, float ext_factor, float attn_factor, rope_corr_dims corr_dims, cudaStream_t stream
|
||||||
) {
|
) {
|
||||||
|
|
||||||
rope_neox_cuda<float>(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
rope_neox_cuda<float>(x, dst, ncols, n_dims, nrows, pos, freq_scale, p_delta_rows, freq_base, ext_factor, attn_factor, corr_dims, stream);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const ggml_tensor * src1 = dst->src[1];
|
const ggml_tensor * src1 = dst->src[1];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
const float * src1_d = (const float *)src1->data;
|
const float * src1_d = (const float *)src1->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
||||||
GGML_ASSERT(src0->type == dst->type);
|
GGML_ASSERT(src0->type == dst->type);
|
||||||
|
|
||||||
const int64_t ne00 = src0->ne[0];
|
const int64_t ne00 = src0->ne[0];
|
||||||
const int64_t ne01 = src0->ne[1];
|
const int64_t ne01 = src0->ne[1];
|
||||||
const int64_t ne2 = dst->ne[2];
|
const int64_t ne2 = dst->ne[2];
|
||||||
const int64_t nrows = ggml_nrows(src0);
|
const int64_t nrows = ggml_nrows(src0);
|
||||||
|
|
||||||
//const int n_past = ((int32_t *) dst->op_params)[0];
|
//const int n_past = ((int32_t *) dst->op_params)[0];
|
||||||
const int n_dims = ((int32_t *) dst->op_params)[1];
|
const int n_dims = ((int32_t *) dst->op_params)[1];
|
||||||
const int mode = ((int32_t *) dst->op_params)[2];
|
const int mode = ((int32_t *) dst->op_params)[2];
|
||||||
const int n_ctx = ((int32_t *) dst->op_params)[3];
|
const int n_ctx = ((int32_t *) dst->op_params)[3];
|
||||||
const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
|
const int n_orig_ctx = ((int32_t *) dst->op_params)[4];
|
||||||
|
|
||||||
// RoPE alteration for extended context
|
// RoPE alteration for extended context
|
||||||
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
float freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow;
|
||||||
memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
|
memcpy(&freq_base, (int32_t *) dst->op_params + 5, sizeof(float));
|
||||||
memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
|
memcpy(&freq_scale, (int32_t *) dst->op_params + 6, sizeof(float));
|
||||||
memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
|
memcpy(&ext_factor, (int32_t *) dst->op_params + 7, sizeof(float));
|
||||||
memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
|
memcpy(&attn_factor, (int32_t *) dst->op_params + 8, sizeof(float));
|
||||||
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
|
memcpy(&beta_fast, (int32_t *) dst->op_params + 9, sizeof(float));
|
||||||
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
|
memcpy(&beta_slow, (int32_t *) dst->op_params + 10, sizeof(float));
|
||||||
|
|
||||||
const int32_t * pos = nullptr;
|
const int32_t * pos = nullptr;
|
||||||
if ((mode & 1) == 0) {
|
if ((mode & 1) == 0) {
|
||||||
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
GGML_ASSERT(src1->type == GGML_TYPE_I32);
|
||||||
GGML_ASSERT(src1->ne[0] == ne2);
|
GGML_ASSERT(src1->ne[0] == ne2);
|
||||||
pos = (const int32_t *) src1_d;
|
pos = (const int32_t *) src1_d;
|
||||||
}
|
}
|
||||||
|
|
||||||
const bool is_neox = mode & 2;
|
const bool is_neox = mode & 2;
|
||||||
const bool is_glm = mode & 4;
|
const bool is_glm = mode & 4;
|
||||||
|
|
||||||
rope_corr_dims corr_dims;
|
rope_corr_dims corr_dims;
|
||||||
ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims.v);
|
ggml_rope_yarn_corr_dims(n_dims, n_orig_ctx, freq_base, beta_fast, beta_slow, corr_dims.v);
|
||||||
|
|
||||||
// compute
|
// compute
|
||||||
if (is_glm) {
|
if (is_glm) {
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
rope_glm_f32_cuda(src0_d, dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, n_ctx, stream);
|
rope_glm_f32_cuda(src0_d, dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, n_ctx, stream);
|
||||||
} else if (is_neox) {
|
} else if (is_neox) {
|
||||||
if (src0->type == GGML_TYPE_F32) {
|
if (src0->type == GGML_TYPE_F32) {
|
||||||
rope_neox_cuda_f32(
|
rope_neox_cuda_f32(
|
||||||
(const float *)src0_d, (float *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
(const float *)src0_d, (float *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
||||||
attn_factor, corr_dims, stream
|
attn_factor, corr_dims, stream
|
||||||
);
|
);
|
||||||
} else if (src0->type == GGML_TYPE_F16) {
|
} else if (src0->type == GGML_TYPE_F16) {
|
||||||
rope_neox_cuda_f16(
|
rope_neox_cuda_f16(
|
||||||
(const half *)src0_d, (half *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
(const half *)src0_d, (half *)dst_d, ne00, n_dims, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
||||||
attn_factor, corr_dims, stream
|
attn_factor, corr_dims, stream
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if (src0->type == GGML_TYPE_F32) {
|
if (src0->type == GGML_TYPE_F32) {
|
||||||
rope_cuda_f32(
|
rope_cuda_f32(
|
||||||
(const float *)src0_d, (float *)dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
(const float *)src0_d, (float *)dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
||||||
attn_factor, corr_dims, stream
|
attn_factor, corr_dims, stream
|
||||||
);
|
);
|
||||||
} else if (src0->type == GGML_TYPE_F16) {
|
} else if (src0->type == GGML_TYPE_F16) {
|
||||||
rope_cuda_f16(
|
rope_cuda_f16(
|
||||||
(const half *)src0_d, (half *)dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
(const half *)src0_d, (half *)dst_d, ne00, nrows, pos, freq_scale, ne01, freq_base, ext_factor,
|
||||||
attn_factor, corr_dims, stream
|
attn_factor, corr_dims, stream
|
||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
GGML_ASSERT(false);
|
GGML_ASSERT(false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/rope.cuh
vendored
10
llama/ggml-cuda/rope.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_ROPE_BLOCK_SIZE 256
|
#define CUDA_ROPE_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
10
llama/ggml-cuda/scale.cuh
vendored
10
llama/ggml-cuda/scale.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_SCALE_BLOCK_SIZE 256
|
#define CUDA_SCALE_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_scale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
10
llama/ggml-cuda/softmax.cuh
vendored
10
llama/ggml-cuda/softmax.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_SOFT_MAX_BLOCK_SIZE 1024
|
#define CUDA_SOFT_MAX_BLOCK_SIZE 1024
|
||||||
|
|
||||||
void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_soft_max(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
80
llama/ggml-cuda/sumrows.cu
vendored
80
llama/ggml-cuda/sumrows.cu
vendored
@ -1,40 +1,40 @@
|
|||||||
#include "sumrows.cuh"
|
#include "sumrows.cuh"
|
||||||
|
|
||||||
static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {
|
static __global__ void k_sum_rows_f32(const float * x, float * dst, const int ncols) {
|
||||||
const int row = blockIdx.x;
|
const int row = blockIdx.x;
|
||||||
const int col = threadIdx.x;
|
const int col = threadIdx.x;
|
||||||
|
|
||||||
float sum = 0.0f;
|
float sum = 0.0f;
|
||||||
for (int i = col; i < ncols; i += blockDim.x) {
|
for (int i = col; i < ncols; i += blockDim.x) {
|
||||||
sum += x[row * ncols + i];
|
sum += x[row * ncols + i];
|
||||||
}
|
}
|
||||||
|
|
||||||
sum = warp_reduce_sum(sum);
|
sum = warp_reduce_sum(sum);
|
||||||
|
|
||||||
if (col == 0) {
|
if (col == 0) {
|
||||||
dst[row] = sum;
|
dst[row] = sum;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
|
static void sum_rows_f32_cuda(const float * x, float * dst, const int ncols, const int nrows, cudaStream_t stream) {
|
||||||
const dim3 block_dims(WARP_SIZE, 1, 1);
|
const dim3 block_dims(WARP_SIZE, 1, 1);
|
||||||
const dim3 block_nums(nrows, 1, 1);
|
const dim3 block_nums(nrows, 1, 1);
|
||||||
k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);
|
k_sum_rows_f32<<<block_nums, block_dims, 0, stream>>>(x, dst, ncols);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(ggml_is_contiguous(src0));
|
GGML_ASSERT(ggml_is_contiguous(src0));
|
||||||
|
|
||||||
|
|
||||||
const int64_t ncols = src0->ne[0];
|
const int64_t ncols = src0->ne[0];
|
||||||
const int64_t nrows = ggml_nrows(src0);
|
const int64_t nrows = ggml_nrows(src0);
|
||||||
|
|
||||||
sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream);
|
sum_rows_f32_cuda(src0_d, dst_d, ncols, nrows, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
6
llama/ggml-cuda/sumrows.cuh
vendored
6
llama/ggml-cuda/sumrows.cuh
vendored
@ -1,3 +1,3 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_sum_rows(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
94
llama/ggml-cuda/tsembd.cu
vendored
94
llama/ggml-cuda/tsembd.cu
vendored
@ -1,47 +1,47 @@
|
|||||||
#include "tsembd.cuh"
|
#include "tsembd.cuh"
|
||||||
|
|
||||||
static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {
|
static __global__ void timestep_embedding_f32(const float * timesteps, float * dst, const int nb1, const int dim, const int max_period) {
|
||||||
// blockIDx.y: idx of timesteps->ne[0]
|
// blockIDx.y: idx of timesteps->ne[0]
|
||||||
// blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE
|
// blockIDx.x: idx of ((dim + 1) / 2) / BLOCK_SIZE
|
||||||
int i = blockIdx.y;
|
int i = blockIdx.y;
|
||||||
int j = threadIdx.x + blockIdx.x * blockDim.x;
|
int j = threadIdx.x + blockIdx.x * blockDim.x;
|
||||||
float * embed_data = (float *)((char *)dst + i*nb1);
|
float * embed_data = (float *)((char *)dst + i*nb1);
|
||||||
|
|
||||||
if (dim % 2 != 0 && j == ((dim + 1) / 2)) {
|
if (dim % 2 != 0 && j == ((dim + 1) / 2)) {
|
||||||
embed_data[dim] = 0.f;
|
embed_data[dim] = 0.f;
|
||||||
}
|
}
|
||||||
|
|
||||||
int half = dim / 2;
|
int half = dim / 2;
|
||||||
if (j >= half) {
|
if (j >= half) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
float timestep = timesteps[i];
|
float timestep = timesteps[i];
|
||||||
float freq = (float)expf(-logf(max_period) * j / half);
|
float freq = (float)expf(-logf(max_period) * j / half);
|
||||||
float arg = timestep * freq;
|
float arg = timestep * freq;
|
||||||
embed_data[j] = cosf(arg);
|
embed_data[j] = cosf(arg);
|
||||||
embed_data[j + half] = sinf(arg);
|
embed_data[j + half] = sinf(arg);
|
||||||
}
|
}
|
||||||
|
|
||||||
static void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1,
|
static void timestep_embedding_f32_cuda(const float * x, float * dst, const int ne00, const int nb1,
|
||||||
const int dim, const int max_period, cudaStream_t stream) {
|
const int dim, const int max_period, cudaStream_t stream) {
|
||||||
int half_ceil = (dim + 1) / 2;
|
int half_ceil = (dim + 1) / 2;
|
||||||
int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE;
|
int num_blocks = (half_ceil + CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE - 1) / CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE;
|
||||||
dim3 gridDim(num_blocks, ne00, 1);
|
dim3 gridDim(num_blocks, ne00, 1);
|
||||||
timestep_embedding_f32<<<gridDim, CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE, 0, stream>>>(x, dst, nb1, dim, max_period);
|
timestep_embedding_f32<<<gridDim, CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE, 0, stream>>>(x, dst, nb1, dim, max_period);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||||
const ggml_tensor * src0 = dst->src[0];
|
const ggml_tensor * src0 = dst->src[0];
|
||||||
const float * src0_d = (const float *)src0->data;
|
const float * src0_d = (const float *)src0->data;
|
||||||
float * dst_d = (float *)dst->data;
|
float * dst_d = (float *)dst->data;
|
||||||
cudaStream_t stream = ctx.stream();
|
cudaStream_t stream = ctx.stream();
|
||||||
|
|
||||||
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
||||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||||
|
|
||||||
const int dim = dst->op_params[0];
|
const int dim = dst->op_params[0];
|
||||||
const int max_period = dst->op_params[1];
|
const int max_period = dst->op_params[1];
|
||||||
|
|
||||||
timestep_embedding_f32_cuda(src0_d, dst_d, src0->ne[0], dst->nb[1], dim, max_period, stream);
|
timestep_embedding_f32_cuda(src0_d, dst_d, src0->ne[0], dst->nb[1], dim, max_period, stream);
|
||||||
}
|
}
|
||||||
|
|||||||
10
llama/ggml-cuda/tsembd.cuh
vendored
10
llama/ggml-cuda/tsembd.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256
|
#define CUDA_TIMESTEP_EMBEDDING_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_timestep_embedding(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
10
llama/ggml-cuda/upscale.cuh
vendored
10
llama/ggml-cuda/upscale.cuh
vendored
@ -1,5 +1,5 @@
|
|||||||
#include "common.cuh"
|
#include "common.cuh"
|
||||||
|
|
||||||
#define CUDA_UPSCALE_BLOCK_SIZE 256
|
#define CUDA_UPSCALE_BLOCK_SIZE 256
|
||||||
|
|
||||||
void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
void ggml_cuda_op_upscale(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||||
|
|||||||
2560
llama/ggml-cuda/vecdotq.cuh
vendored
2560
llama/ggml-cuda/vecdotq.cuh
vendored
File diff suppressed because it is too large
Load Diff
132
llama/ggml-metal.h
vendored
132
llama/ggml-metal.h
vendored
@ -1,66 +1,66 @@
|
|||||||
// An interface allowing to compute ggml_cgraph with Metal
|
// An interface allowing to compute ggml_cgraph with Metal
|
||||||
//
|
//
|
||||||
// This is a fully functional interface that extends ggml with GPU support for Apple devices.
|
// This is a fully functional interface that extends ggml with GPU support for Apple devices.
|
||||||
// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
|
// A similar interface can be created for other GPU backends (e.g. Vulkan, CUDA, OpenCL, etc.)
|
||||||
//
|
//
|
||||||
// How it works?
|
// How it works?
|
||||||
//
|
//
|
||||||
// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
|
// As long as your program can create and evaluate a ggml_cgraph on the CPU, you can use this
|
||||||
// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
|
// interface to evaluate the same graph on the GPU. Instead of using ggml_graph_compute(), you
|
||||||
// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
|
// use ggml_metal_graph_compute() (or ggml_vulkan_graph_compute(), etc.)
|
||||||
//
|
//
|
||||||
// You only need to make sure that all memory buffers that you used during the graph creation
|
// You only need to make sure that all memory buffers that you used during the graph creation
|
||||||
// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
|
// are mapped to the device memory with the ggml_metal_add_buffer() function. This mapping is
|
||||||
// used during the graph evaluation to determine the arguments of the compute kernels.
|
// used during the graph evaluation to determine the arguments of the compute kernels.
|
||||||
//
|
//
|
||||||
// Synchronization between device and host memory (for example for input and output tensors)
|
// Synchronization between device and host memory (for example for input and output tensors)
|
||||||
// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
|
// is done with the ggml_metal_set_tensor() and ggml_metal_get_tensor() functions.
|
||||||
//
|
//
|
||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
#include "ggml-backend.h"
|
#include "ggml-backend.h"
|
||||||
|
|
||||||
#include <stddef.h>
|
#include <stddef.h>
|
||||||
#include <stdbool.h>
|
#include <stdbool.h>
|
||||||
|
|
||||||
// max memory buffers that can be mapped to the device
|
// max memory buffers that can be mapped to the device
|
||||||
#define GGML_METAL_MAX_BUFFERS 64
|
#define GGML_METAL_MAX_BUFFERS 64
|
||||||
|
|
||||||
struct ggml_tensor;
|
struct ggml_tensor;
|
||||||
struct ggml_cgraph;
|
struct ggml_cgraph;
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
//
|
//
|
||||||
// backend API
|
// backend API
|
||||||
// user-code should use only these functions
|
// user-code should use only these functions
|
||||||
//
|
//
|
||||||
|
|
||||||
GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);
|
GGML_API void ggml_backend_metal_log_set_callback(ggml_log_callback log_callback, void * user_data);
|
||||||
|
|
||||||
GGML_API ggml_backend_t ggml_backend_metal_init(void);
|
GGML_API ggml_backend_t ggml_backend_metal_init(void);
|
||||||
|
|
||||||
GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);
|
GGML_API bool ggml_backend_is_metal(ggml_backend_t backend);
|
||||||
|
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);
|
GGML_API GGML_CALL ggml_backend_buffer_t ggml_backend_metal_buffer_from_ptr(void * data, size_t size, size_t max_size);
|
||||||
|
|
||||||
GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
|
GGML_API void ggml_backend_metal_set_n_cb(ggml_backend_t backend, int n_cb);
|
||||||
|
|
||||||
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
|
GGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_metal_buffer_type(void);
|
||||||
|
|
||||||
// helper to check if the device supports a specific family
|
// helper to check if the device supports a specific family
|
||||||
// ideally, the user code should be doing these checks
|
// ideally, the user code should be doing these checks
|
||||||
// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
|
// ref: https://developer.apple.com/metal/Metal-Feature-Set-Tables.pdf
|
||||||
GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);
|
GGML_API bool ggml_backend_metal_supports_family(ggml_backend_t backend, int family);
|
||||||
|
|
||||||
// capture all command buffers committed the next time `ggml_backend_graph_compute` is called
|
// capture all command buffers committed the next time `ggml_backend_graph_compute` is called
|
||||||
GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
|
GGML_API void ggml_backend_metal_capture_next_compute(ggml_backend_t backend);
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|||||||
6859
llama/ggml-metal.metal
Normal file
6859
llama/ggml-metal.metal
Normal file
File diff suppressed because it is too large
Load Diff
BIN
llama/ggml-metal.o
Normal file
BIN
llama/ggml-metal.o
Normal file
Binary file not shown.
266
llama/ggml-quants.h
vendored
266
llama/ggml-quants.h
vendored
@ -1,133 +1,133 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#define GGML_COMMON_DECL_C
|
#define GGML_COMMON_DECL_C
|
||||||
#include "ggml-common.h"
|
#include "ggml-common.h"
|
||||||
|
|
||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
|
|
||||||
// GGML internal header
|
// GGML internal header
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
extern "C" {
|
extern "C" {
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
// Quantization
|
// Quantization
|
||||||
void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_0_reference(const float * GGML_RESTRICT x, block_q4_0 * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_1_reference(const float * GGML_RESTRICT x, block_q4_1 * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_0_reference(const float * GGML_RESTRICT x, block_q5_0 * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_1_reference(const float * GGML_RESTRICT x, block_q5_1 * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_0_reference(const float * GGML_RESTRICT x, block_q8_0 * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_1_reference(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q2_K_reference(const float * GGML_RESTRICT x, block_q2_K * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q3_K_reference(const float * GGML_RESTRICT x, block_q3_K * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_K_reference(const float * GGML_RESTRICT x, block_q4_K * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_K_reference(const float * GGML_RESTRICT x, block_q5_K * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q6_K_reference(const float * GGML_RESTRICT x, block_q6_K * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_K_reference(const float * GGML_RESTRICT x, block_q8_K * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq3_xxs_reference(const float * GGML_RESTRICT x, block_iq3_xxs * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq4_nl_reference (const float * GGML_RESTRICT x, block_iq4_nl * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq4_xs_reference (const float * GGML_RESTRICT x, block_iq4_xs * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq3_s_reference (const float * GGML_RESTRICT x, block_iq3_s * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq2_s_reference (const float * GGML_RESTRICT x, block_iq2_s * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_0(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_1(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q2_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q3_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q4_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q5_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q6_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_q8_K(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq3_xxs(const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq4_nl (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq4_xs (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq3_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
void quantize_row_iq2_s (const float * GGML_RESTRICT x, void * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
// Dequantization
|
// Dequantization
|
||||||
void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q4_0(const block_q4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q4_1(const block_q4_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q5_0(const block_q5_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q5_1(const block_q5_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
//void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q2_K(const block_q2_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q3_K(const block_q3_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q4_K(const block_q4_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q5_K(const block_q5_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q6_K(const block_q6_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_q8_K(const block_q8_K * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq2_xxs(const block_iq2_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq2_xs (const block_iq2_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||||
|
|
||||||
// Dot product
|
// Dot product
|
||||||
void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q5_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q8_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
|
|
||||||
void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q2_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q3_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
|
|
||||||
void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq2_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq2_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||||
|
|
||||||
// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
|
// Quantization utilizing an importance matrix (a.k.a. "Activation aWare Quantization")
|
||||||
size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq2_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq2_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
|
|
||||||
size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q2_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q3_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q4_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q5_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q6_K(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q4_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q5_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||||
|
|
||||||
void iq2xs_init_impl(enum ggml_type type);
|
void iq2xs_init_impl(enum ggml_type type);
|
||||||
void iq2xs_free_impl(enum ggml_type type);
|
void iq2xs_free_impl(enum ggml_type type);
|
||||||
void iq3xs_init_impl(int grid_size);
|
void iq3xs_init_impl(int grid_size);
|
||||||
void iq3xs_free_impl(int grid_size);
|
void iq3xs_free_impl(int grid_size);
|
||||||
|
|
||||||
#ifdef __cplusplus
|
#ifdef __cplusplus
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
|||||||
@ -3,14 +3,13 @@ package llama
|
|||||||
// #cgo darwin,arm64 CFLAGS: -std=c11 -DGGML_USE_METAL -DGGML_METAL_EMBED_LIBRARY -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
|
// #cgo darwin,arm64 CFLAGS: -std=c11 -DGGML_USE_METAL -DGGML_METAL_EMBED_LIBRARY -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
|
||||||
// #cgo darwin,arm64 CXXFLAGS: -std=c++11 -DGGML_USE_METAL -DGGML_METAL_EMBED_LIBRARY -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
|
// #cgo darwin,arm64 CXXFLAGS: -std=c++11 -DGGML_USE_METAL -DGGML_METAL_EMBED_LIBRARY -DGGML_USE_ACCELERATE -DACCELERATE_NEW_LAPACK -DACCELERATE_LAPACK_ILP64
|
||||||
// #cgo darwin,amd64 CXXFLAGS: -std=c++11
|
// #cgo darwin,amd64 CXXFLAGS: -std=c++11
|
||||||
// #cgo darwin,arm64 LDFLAGS: ggml-metal.o -framework Foundation -framework Metal -framework MetalKit -framework Accelerate
|
// #cgo darwin,arm64 LDFLAGS: -ld_classic ${SRCDIR}/ggml-metal.o -framework Foundation -framework Metal -framework MetalKit -framework Accelerate
|
||||||
// #cgo darwin,amd64 LDFLAGS: -framework Foundation -framework Accelerate
|
// #cgo darwin,amd64 LDFLAGS: -ld_classic -framework Foundation -framework Accelerate
|
||||||
// #cgo windows LDFLAGS: -lmsvcrt
|
// #cgo windows LDFLAGS: -lmsvcrt
|
||||||
// #cgo avx CFLAGS: -mavx
|
// #cgo avx CFLAGS: -mavx
|
||||||
// #cgo avx CXXFLAGS: -mavx
|
// #cgo avx CXXFLAGS: -mavx
|
||||||
// #cgo avx2 CFLAGS: -mavx -mavx2 -mfma
|
// #cgo avx2 CFLAGS: -mavx -mavx2 -mfma
|
||||||
// #cgo avx2 CXXFLAGS: -mavx -mavx2 -mfma
|
// #cgo avx2 CXXFLAGS: -mavx -mavx2 -mfma
|
||||||
// #cgo avx2 LDFLAGS: -lm
|
|
||||||
// #cgo cuda CFLAGS: -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
// #cgo cuda CFLAGS: -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
||||||
// #cgo cuda CXXFLAGS: -std=c++11 -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
// #cgo cuda CXXFLAGS: -std=c++11 -DGGML_USE_CUDA -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
||||||
// #cgo rocm CXXFLAGS: -std=c++11 -DGGML_USE_CUDA -DGGML_USE_HIPBLAS -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
// #cgo rocm CXXFLAGS: -std=c++11 -DGGML_USE_CUDA -DGGML_USE_HIPBLAS -DGGML_CUDA_DMMV_X=32 -DGGML_CUDA_PEER_MAX_BATCH_SIZE=128 -DGGML_MULTIPLATFORM -DGGML_CUDA_MMV_Y=1 -DGGML_BUILD=1
|
||||||
@ -24,6 +23,8 @@ import (
|
|||||||
"runtime"
|
"runtime"
|
||||||
"strings"
|
"strings"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
|
|
||||||
|
"github.com/ollama/ollama/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type Token int32
|
type Token int32
|
||||||
@ -201,3 +202,21 @@ func (m *Model) Tokenize(text string, maxTokens int, addSpecial bool, parseSpeci
|
|||||||
|
|
||||||
return tokens, nil
|
return tokens, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func Quantize(infile, outfile string, ftype llm.FileType) error {
|
||||||
|
cinfile := C.CString(infile)
|
||||||
|
defer C.free(unsafe.Pointer(cinfile))
|
||||||
|
|
||||||
|
coutfile := C.CString(outfile)
|
||||||
|
defer C.free(unsafe.Pointer(coutfile))
|
||||||
|
|
||||||
|
params := C.llama_model_quantize_default_params()
|
||||||
|
params.nthread = -1
|
||||||
|
params.ftype = ftype.Value()
|
||||||
|
|
||||||
|
if rc := C.llama_model_quantize(cinfile, coutfile, ¶ms); rc != 0 {
|
||||||
|
return fmt.Errorf("llama_model_quantize: %d", rc)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@ -1,11 +0,0 @@
|
|||||||
sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > ggml-metal-embed.metal
|
|
||||||
TEMP_ASSEMBLY=$(mktemp)
|
|
||||||
echo ".section __DATA, __ggml_metallib" > $TEMP_ASSEMBLY
|
|
||||||
echo ".globl _ggml_metallib_start" >> $TEMP_ASSEMBLY
|
|
||||||
echo "_ggml_metallib_start:" >> $TEMP_ASSEMBLY
|
|
||||||
echo ".incbin \"ggml-metal-embed.metal\"" >> $TEMP_ASSEMBLY
|
|
||||||
echo ".globl _ggml_metallib_end" >> $TEMP_ASSEMBLY
|
|
||||||
echo "_ggml_metallib_end:" >> $TEMP_ASSEMBLY
|
|
||||||
as -mmacosx-version-min=11.3 $TEMP_ASSEMBLY -o ggml-metal.o
|
|
||||||
rm -f $TEMP_ASSEMBLY
|
|
||||||
rm -rf ggml-metal-embed.metal
|
|
||||||
@ -5,5 +5,5 @@
|
|||||||
```
|
```
|
||||||
|
|
||||||
```
|
```
|
||||||
curl POST -H "Content-Type: application/json" -d '{"prompt": "hi"}' http://localhost:8080/
|
curl -X POST -H "Content-Type: application/json" -d '{"prompt": "hi"}' http://localhost:8080/
|
||||||
```
|
```
|
||||||
|
|||||||
@ -23,29 +23,9 @@ type Response struct {
|
|||||||
type Server struct {
|
type Server struct {
|
||||||
model *llama.Model
|
model *llama.Model
|
||||||
lc *llama.Context
|
lc *llama.Context
|
||||||
batch *llama.Batch
|
|
||||||
|
|
||||||
queue chan Sequence
|
|
||||||
seqs []*Sequence
|
|
||||||
|
|
||||||
// mu guards seqs
|
|
||||||
mu sync.Mutex
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type Sequence struct {
|
var mu sync.Mutex
|
||||||
prompt []llama.Token
|
|
||||||
out chan string
|
|
||||||
}
|
|
||||||
|
|
||||||
func schedule(parallel int, queue <-chan Sequence) {
|
|
||||||
// Fill sequences from the queue
|
|
||||||
|
|
||||||
// once a sequence finishes, remove it from and add a new one from the queue
|
|
||||||
}
|
|
||||||
|
|
||||||
func process() {
|
|
||||||
// loop through the sequences, fill a batch, decode and sample tokens, responding to appropriate requests
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) stream(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) stream(w http.ResponseWriter, r *http.Request) {
|
||||||
var request Request
|
var request Request
|
||||||
@ -59,23 +39,15 @@ func (s *Server) stream(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("Transfer-Encoding", "chunked")
|
w.Header().Set("Transfer-Encoding", "chunked")
|
||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
|
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
|
||||||
|
// main loop
|
||||||
tokens, err := s.model.Tokenize(request.Prompt, 2048, true, true)
|
tokens, err := s.model.Tokenize(request.Prompt, 2048, true, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
seq := Sequence{prompt: tokens}
|
batch := llama.NewBatch(512, 0, 1)
|
||||||
s.queue <- seq
|
|
||||||
|
|
||||||
// listen for the sequence to finish
|
|
||||||
for {
|
|
||||||
str := <-seq.out
|
|
||||||
if err := json.NewEncoder(w).Encode(&Response{Token: str}); err != nil {
|
|
||||||
log.Println("Failed to encode result:", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.(http.Flusher).Flush()
|
|
||||||
}
|
|
||||||
|
|
||||||
// prompt eval
|
// prompt eval
|
||||||
for i, t := range tokens {
|
for i, t := range tokens {
|
||||||
@ -115,7 +87,6 @@ func (s *Server) stream(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
mp := flag.String("model", "", "Path to model binary file")
|
mp := flag.String("model", "", "Path to model binary file")
|
||||||
parallel := flag.Int("parallel", 1, "Number of parallel requests to handle")
|
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// load the model
|
// load the model
|
||||||
@ -131,8 +102,6 @@ func main() {
|
|||||||
server := &Server{
|
server := &Server{
|
||||||
model: model,
|
model: model,
|
||||||
lc: lc,
|
lc: lc,
|
||||||
queue: make(chan Sequence, 256),
|
|
||||||
seqs: make([]*Sequence, *parallel),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
addr := "127.0.0.1:8080"
|
addr := "127.0.0.1:8080"
|
||||||
@ -23,7 +23,7 @@ cp $src_dir/ggml-quants.c $dst_dir/ggml-quants.c
|
|||||||
cp $src_dir/ggml-quants.h $dst_dir/ggml-quants.h
|
cp $src_dir/ggml-quants.h $dst_dir/ggml-quants.h
|
||||||
cp $src_dir/ggml-metal.metal $dst_dir/ggml-metal.metal
|
cp $src_dir/ggml-metal.metal $dst_dir/ggml-metal.metal
|
||||||
cp $src_dir/ggml-metal.h $dst_dir/ggml-metal.h
|
cp $src_dir/ggml-metal.h $dst_dir/ggml-metal.h
|
||||||
cp $src_dir/ggml-metal.m $dst_dir/ggml-metal-darwin_arm64.m
|
cp $src_dir/ggml-metal.m $dst_dir/ggml-metal.m
|
||||||
cp $src_dir/ggml-impl.h $dst_dir/ggml-impl.h
|
cp $src_dir/ggml-impl.h $dst_dir/ggml-impl.h
|
||||||
cp $src_dir/ggml-cuda.h $dst_dir/ggml-cuda.h
|
cp $src_dir/ggml-cuda.h $dst_dir/ggml-cuda.h
|
||||||
cp $src_dir/ggml-cuda.cu $dst_dir/ggml-cuda.cu
|
cp $src_dir/ggml-cuda.cu $dst_dir/ggml-cuda.cu
|
||||||
@ -34,11 +34,23 @@ cp $src_dir/ggml-backend-impl.h $dst_dir/ggml-backend-impl.h
|
|||||||
cp $src_dir/ggml-alloc.h $dst_dir/ggml-alloc.h
|
cp $src_dir/ggml-alloc.h $dst_dir/ggml-alloc.h
|
||||||
cp $src_dir/ggml-alloc.c $dst_dir/ggml-alloc.c
|
cp $src_dir/ggml-alloc.c $dst_dir/ggml-alloc.c
|
||||||
|
|
||||||
sed -i 's/extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();/\/\/ extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();/' ggml-cuda.cu
|
|
||||||
sed -i '34iGGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);' ggml-cuda.h
|
|
||||||
|
|
||||||
|
|
||||||
# ggml-cuda
|
# ggml-cuda
|
||||||
mkdir -p $dst_dir/ggml-cuda
|
mkdir -p $dst_dir/ggml-cuda
|
||||||
cp $src_dir/ggml-cuda/*.cu $dst_dir/ggml-cuda/
|
cp $src_dir/ggml-cuda/*.cu $dst_dir/ggml-cuda/
|
||||||
cp $src_dir/ggml-cuda/*.cuh $dst_dir/ggml-cuda/
|
cp $src_dir/ggml-cuda/*.cuh $dst_dir/ggml-cuda/
|
||||||
|
|
||||||
|
sed -i 's/extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();/\/\/ extern "C" GGML_CALL int ggml_backend_cuda_reg_devices();/' ggml-cuda.cu
|
||||||
|
sed -i '34iGGML_API GGML_CALL ggml_backend_buffer_type_t ggml_backend_cuda_host_buffer_type(void);' ggml-cuda.h
|
||||||
|
|
||||||
|
# ggml-metal
|
||||||
|
sed -e '/#include "ggml-common.h"/r ggml-common.h' -e '/#include "ggml-common.h"/d' < ggml-metal.metal > temp.metal
|
||||||
|
TEMP_ASSEMBLY=$(mktemp)
|
||||||
|
echo ".section __DATA, __ggml_metallib" > $TEMP_ASSEMBLY
|
||||||
|
echo ".globl _ggml_metallib_start" >> $TEMP_ASSEMBLY
|
||||||
|
echo "_ggml_metallib_start:" >> $TEMP_ASSEMBLY
|
||||||
|
echo ".incbin \"temp.metal\"" >> $TEMP_ASSEMBLY
|
||||||
|
echo ".globl _ggml_metallib_end" >> $TEMP_ASSEMBLY
|
||||||
|
echo "_ggml_metallib_end:" >> $TEMP_ASSEMBLY
|
||||||
|
as -mmacosx-version-min=11.3 $TEMP_ASSEMBLY -o ggml-metal.o
|
||||||
|
rm -f $TEMP_ASSEMBLY
|
||||||
|
rm -rf temp.metal
|
||||||
|
|||||||
@ -2,10 +2,10 @@ package llm
|
|||||||
|
|
||||||
import "fmt"
|
import "fmt"
|
||||||
|
|
||||||
type fileType uint32
|
type FileType uint32
|
||||||
|
|
||||||
const (
|
const (
|
||||||
fileTypeF32 fileType = iota
|
fileTypeF32 FileType = iota
|
||||||
fileTypeF16
|
fileTypeF16
|
||||||
fileTypeQ4_0
|
fileTypeQ4_0
|
||||||
fileTypeQ4_1
|
fileTypeQ4_1
|
||||||
@ -41,7 +41,7 @@ const (
|
|||||||
fileTypeUnknown
|
fileTypeUnknown
|
||||||
)
|
)
|
||||||
|
|
||||||
func ParseFileType(s string) (fileType, error) {
|
func ParseFileType(s string) (FileType, error) {
|
||||||
switch s {
|
switch s {
|
||||||
case "F32":
|
case "F32":
|
||||||
return fileTypeF32, nil
|
return fileTypeF32, nil
|
||||||
@ -108,7 +108,7 @@ func ParseFileType(s string) (fileType, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t fileType) String() string {
|
func (t FileType) String() string {
|
||||||
switch t {
|
switch t {
|
||||||
case fileTypeF32:
|
case fileTypeF32:
|
||||||
return "F32"
|
return "F32"
|
||||||
@ -175,6 +175,6 @@ func (t fileType) String() string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t fileType) Value() uint32 {
|
func (t FileType) Value() uint32 {
|
||||||
return uint32(t)
|
return uint32(t)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -100,4 +100,4 @@ esac
|
|||||||
|
|
||||||
cleanup
|
cleanup
|
||||||
wait_for_compress
|
wait_for_compress
|
||||||
echo "go generate completed. LLM runners: $(cd ${BUILD_DIR}/..; echo *)"
|
echo "go generate completed. LLM runners: $(cd ${BUILD_DIR}/..; echo *)"
|
||||||
@ -58,19 +58,6 @@ init_vars
|
|||||||
git_module_setup
|
git_module_setup
|
||||||
apply_patches
|
apply_patches
|
||||||
|
|
||||||
init_vars
|
|
||||||
if [ -z "${OLLAMA_SKIP_STATIC_GENERATE}" -o "${OLLAMA_CPU_TARGET}" = "static" ]; then
|
|
||||||
# Builds by default, allows skipping, forces build if OLLAMA_CPU_TARGET="static"
|
|
||||||
# Enables optimized Dockerfile builds using a blanket skip and targeted overrides
|
|
||||||
# Static build for linking into the Go binary
|
|
||||||
init_vars
|
|
||||||
CMAKE_TARGETS="--target llama --target ggml"
|
|
||||||
CMAKE_DEFS="-DBUILD_SHARED_LIBS=off -DGGML_NATIVE=off -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_OPENMP=off ${CMAKE_DEFS}"
|
|
||||||
BUILD_DIR="../build/linux/${ARCH}_static"
|
|
||||||
echo "Building static library"
|
|
||||||
build
|
|
||||||
fi
|
|
||||||
|
|
||||||
init_vars
|
init_vars
|
||||||
if [ -z "${OLLAMA_SKIP_CPU_GENERATE}" ]; then
|
if [ -z "${OLLAMA_SKIP_CPU_GENERATE}" ]; then
|
||||||
# Users building from source can tune the exact flags we pass to cmake for configuring
|
# Users building from source can tune the exact flags we pass to cmake for configuring
|
||||||
|
|||||||
@ -177,40 +177,6 @@ function cleanup {
|
|||||||
# -DGGML_AVX2 -- 2013 Intel Haswell & 2015 AMD Excavator / 2017 AMD Zen
|
# -DGGML_AVX2 -- 2013 Intel Haswell & 2015 AMD Excavator / 2017 AMD Zen
|
||||||
# -DGGML_FMA (FMA3) -- 2013 Intel Haswell & 2012 AMD Piledriver
|
# -DGGML_FMA (FMA3) -- 2013 Intel Haswell & 2012 AMD Piledriver
|
||||||
|
|
||||||
|
|
||||||
function build_static() {
|
|
||||||
if ((-not "${env:OLLAMA_SKIP_STATIC_GENERATE}") -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "static"))) {
|
|
||||||
# GCC build for direct linking into the Go binary
|
|
||||||
init_vars
|
|
||||||
# cmake will silently fallback to msvc compilers if mingw isn't in the path, so detect and fail fast
|
|
||||||
# as we need this to be compiled by gcc for golang to be able to link with itx
|
|
||||||
write-host "Checking for MinGW..."
|
|
||||||
# error action ensures we exit on failure
|
|
||||||
get-command gcc
|
|
||||||
get-command mingw32-make
|
|
||||||
$oldTargets = $script:cmakeTargets
|
|
||||||
$script:cmakeTargets = @("llama", "ggml")
|
|
||||||
$script:cmakeDefs = @(
|
|
||||||
"-G", "MinGW Makefiles"
|
|
||||||
"-DCMAKE_C_COMPILER=gcc.exe",
|
|
||||||
"-DCMAKE_CXX_COMPILER=g++.exe",
|
|
||||||
"-DBUILD_SHARED_LIBS=off",
|
|
||||||
"-DGGML_NATIVE=off",
|
|
||||||
"-DGGML_AVX=off",
|
|
||||||
"-DGGML_AVX2=off",
|
|
||||||
"-DGGML_AVX512=off",
|
|
||||||
"-DGGML_F16C=off",
|
|
||||||
"-DGGML_FMA=off",
|
|
||||||
"-DGGML_OPENMP=off")
|
|
||||||
$script:buildDir="../build/windows/${script:ARCH}_static"
|
|
||||||
write-host "Building static library"
|
|
||||||
build
|
|
||||||
$script:cmakeTargets = $oldTargets
|
|
||||||
} else {
|
|
||||||
write-host "Skipping CPU generation step as requested"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function build_cpu($gen_arch) {
|
function build_cpu($gen_arch) {
|
||||||
if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
|
if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
|
||||||
# remaining llama.cpp builds use MSVC
|
# remaining llama.cpp builds use MSVC
|
||||||
@ -398,7 +364,6 @@ init_vars
|
|||||||
if ($($args.count) -eq 0) {
|
if ($($args.count) -eq 0) {
|
||||||
git_module_setup
|
git_module_setup
|
||||||
apply_patches
|
apply_patches
|
||||||
build_static
|
|
||||||
if ($script:ARCH -eq "arm64") {
|
if ($script:ARCH -eq "arm64") {
|
||||||
build_cpu("ARM64")
|
build_cpu("ARM64")
|
||||||
} else { # amd64
|
} else { # amd64
|
||||||
|
|||||||
@ -55,9 +55,9 @@ func (kv KV) ParameterCount() uint64 {
|
|||||||
return kv.u64("general.parameter_count")
|
return kv.u64("general.parameter_count")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (kv KV) FileType() fileType {
|
func (kv KV) FileType() FileType {
|
||||||
if u64 := kv.u64("general.file_type"); u64 > 0 {
|
if u64 := kv.u64("general.file_type"); u64 > 0 {
|
||||||
return fileType(uint32(u64))
|
return FileType(uint32(u64))
|
||||||
}
|
}
|
||||||
|
|
||||||
return fileTypeUnknown
|
return fileTypeUnknown
|
||||||
|
|||||||
41
llm/llm.go
41
llm/llm.go
@ -1,41 +0,0 @@
|
|||||||
package llm
|
|
||||||
|
|
||||||
// #cgo CFLAGS: -Illama.cpp -Illama.cpp/include -Illama.cpp/ggml/include
|
|
||||||
// #cgo LDFLAGS: -lllama -lggml -lstdc++ -lpthread
|
|
||||||
// #cgo darwin,arm64 LDFLAGS: -L${SRCDIR}/build/darwin/arm64_static -L${SRCDIR}/build/darwin/arm64_static/src -L${SRCDIR}/build/darwin/arm64_static/ggml/src -framework Accelerate -framework Metal
|
|
||||||
// #cgo darwin,amd64 LDFLAGS: -L${SRCDIR}/build/darwin/x86_64_static -L${SRCDIR}/build/darwin/x86_64_static/src -L${SRCDIR}/build/darwin/x86_64_static/ggml/src
|
|
||||||
// #cgo windows,amd64 LDFLAGS: -static-libstdc++ -static-libgcc -static -L${SRCDIR}/build/windows/amd64_static -L${SRCDIR}/build/windows/amd64_static/src -L${SRCDIR}/build/windows/amd64_static/ggml/src
|
|
||||||
// #cgo windows,arm64 LDFLAGS: -static-libstdc++ -static-libgcc -static -L${SRCDIR}/build/windows/arm64_static -L${SRCDIR}/build/windows/arm64_static/src -L${SRCDIR}/build/windows/arm64_static/ggml/src
|
|
||||||
// #cgo linux,amd64 LDFLAGS: -L${SRCDIR}/build/linux/x86_64_static -L${SRCDIR}/build/linux/x86_64_static/src -L${SRCDIR}/build/linux/x86_64_static/ggml/src
|
|
||||||
// #cgo linux,arm64 LDFLAGS: -L${SRCDIR}/build/linux/arm64_static -L${SRCDIR}/build/linux/arm64_static/src -L${SRCDIR}/build/linux/arm64_static/ggml/src
|
|
||||||
// #include <stdlib.h>
|
|
||||||
// #include "llama.h"
|
|
||||||
import "C"
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"unsafe"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SystemInfo is an unused example of calling llama.cpp functions using CGo
|
|
||||||
func SystemInfo() string {
|
|
||||||
return C.GoString(C.llama_print_system_info())
|
|
||||||
}
|
|
||||||
|
|
||||||
func Quantize(infile, outfile string, ftype fileType) error {
|
|
||||||
cinfile := C.CString(infile)
|
|
||||||
defer C.free(unsafe.Pointer(cinfile))
|
|
||||||
|
|
||||||
coutfile := C.CString(outfile)
|
|
||||||
defer C.free(unsafe.Pointer(coutfile))
|
|
||||||
|
|
||||||
params := C.llama_model_quantize_default_params()
|
|
||||||
params.nthread = -1
|
|
||||||
params.ftype = ftype.Value()
|
|
||||||
|
|
||||||
if rc := C.llama_model_quantize(cinfile, coutfile, ¶ms); rc != 0 {
|
|
||||||
return errors.New("failed to quantize model. This model architecture may not be supported, or you may need to upgrade Ollama to the latest version")
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
@ -26,6 +26,7 @@ import (
|
|||||||
"github.com/ollama/ollama/auth"
|
"github.com/ollama/ollama/auth"
|
||||||
"github.com/ollama/ollama/envconfig"
|
"github.com/ollama/ollama/envconfig"
|
||||||
"github.com/ollama/ollama/format"
|
"github.com/ollama/ollama/format"
|
||||||
|
"github.com/ollama/ollama/llama"
|
||||||
"github.com/ollama/ollama/llm"
|
"github.com/ollama/ollama/llm"
|
||||||
"github.com/ollama/ollama/parser"
|
"github.com/ollama/ollama/parser"
|
||||||
"github.com/ollama/ollama/template"
|
"github.com/ollama/ollama/template"
|
||||||
@ -453,7 +454,7 @@ func CreateModel(ctx context.Context, name model.Name, modelFileDir, quantizatio
|
|||||||
defer temp.Close()
|
defer temp.Close()
|
||||||
defer os.Remove(temp.Name())
|
defer os.Remove(temp.Name())
|
||||||
|
|
||||||
if err := llm.Quantize(blob, temp.Name(), want); err != nil {
|
if err := llama.Quantize(blob, temp.Name(), want); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user