server

llm/ext_server/server.cpp

@@ -1040,6 +1040,7 @@ struct llama_server_context
             img.request_encode_image = false;
         }
 
+        LOG_TEE("slot has images: %d\n", slot.images.size());
         return slot.images.size() > 0;
     }
 
@@ -1271,6 +1272,71 @@ struct llama_server_context
         }
     }
 
+    bool process_images_paligemma(server_slot &slot, int n_batch)
+    {
+        int n_past = 0;
+        int image_idx = 0;
+        slot_image &img = slot.images[image_idx];
+
+        // rescale image embeddings
+        float *data = img.image_embedding;
+        for (int i = 0; i < 2048 * 256; i++)
+        {
+            data[i] = data[i] / sqrt(2048);
+        }
+
+        set_image_embeds(ctx, data);
+
+        // generate user_prompt -> this should contain image tokens prepended and a new line appended:
+        // batch.n_tokens += (int)slot.images.size() * llama_n_embd(model);
+
+        std::vector<llama_token> tokens;
+        std::string prompt = "What is in this image";
+        std::vector<llama_token> text = ::llama_tokenize(ctx, prompt, false, true);
+
+        for (int i = 0; i < (int)slot.images.size() * 256; i++)
+        {
+            tokens.push_back(257152);
+        }
+
+        tokens.push_back(2);
+
+        printf("btach.n_tokens %d\n", batch.n_tokens);
+
+        for (int i = 0; i < text.size(); i++)
+        {
+            // printf("token [%d]: %d\n", text[i]);
+            tokens.push_back(text[i]);
+        }
+
+        tokens.push_back(108);
+
+        batch.n_tokens = (int)slot.images.size() * 256 + 2 + text.size();
+
+        for (int i = 0; i < batch.n_tokens; i++)
+        {
+            printf("token %d: %d\n", i, tokens[i]);
+        }
+
+        for (int i = 0; i < batch.n_tokens; i += n_batch)
+        {
+            printf("calling decode\n");
+            int n_eval = (int)batch.n_tokens - i;
+            if (n_eval > n_batch)
+            {
+                n_eval = n_batch;
+            }
+            printf("n_eval: %d, n_past: %d", n_eval, n_past);
+            if (llama_decode(ctx, llama_batch_get_one(&tokens[i], n_eval, 0, 0)))
+            {
+                printf("%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, batch.n_tokens, n_batch, n_past);
+                return false;
+            }
+            n_past += n_eval;
+        }
+        return true;
+    }
+
     // for multiple images processing
     bool ingest_images(server_slot &slot, int n_batch)
     {
@@ -1833,12 +1899,17 @@ struct llama_server_context
                         slot_npast++;
                     }
 
-                    if (has_images && !ingest_images(slot, n_batch))
+                    LOG_ERROR("checking has images", {
+                                                         {"has images", has_images},
+                                                         {"task_id", slot.task_id},
+                                                     });
+                    // if (has_images && !ingest_images(slot, n_batch))
+                    if (has_images && !process_images_paligemma(slot, n_batch))
                     {
                         LOG_ERROR("failed processing images", {
-                            {"slot_id", slot.id},
-                            {"task_id", slot.task_id},
-                        });
+                                                                  {"slot_id", slot.id},
+                                                                  {"task_id", slot.task_id},
+                                                              });
                         // FIXME @phymbert: to be properly tested
                         //  early returning without changing the slot state will block the slot for ever
                         // no one at the moment is checking the return value

llm/patches/12-paligemma.diff

@@ -0,0 +1,94 @@
+diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
+index 7cda5f10..50fbcf08 100644
+--- a/examples/llava/clip.cpp
++++ b/examples/llava/clip.cpp
+@@ -709,9 +709,12 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
+             embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
+             embeddings = ggml_add(ctx0, embeddings, model.mm_0_b);
+ 
+-            embeddings = ggml_gelu(ctx0, embeddings);
+-            embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
+-            embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
++            // paligemma missing second linear layer
++            if (model.mm_2_w) {
++                embeddings = ggml_gelu(ctx0, embeddings);
++                embeddings = ggml_mul_mat(ctx0, model.mm_2_w, embeddings);
++                embeddings = ggml_add(ctx0, embeddings, model.mm_2_b);
++            }
+ 
+         } else if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
+             embeddings = ggml_mul_mat(ctx0, model.mm_0_w, embeddings);
+@@ -2076,7 +2079,10 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
+         return ctx->vision_model.mm_model_peg_0_b->ne[0];
+     }
+     if (ctx->proj_type == PROJECTOR_TYPE_MLP) {
+-        return ctx->vision_model.mm_2_b->ne[0];
++        // paligemma missing second linear layer
++        if (ctx->vision_model.mm_2_b == nullptr) {
++            return ctx->vision_model.mm_0_b->ne[0];
++        }
+     }
+     if (ctx->proj_type == PROJECTOR_TYPE_MLP_NORM) {
+         return ctx->vision_model.mm_3_b->ne[0];
+diff --git a/include/llama.h b/include/llama.h
+index f23355a6..7c6301bf 100644
+--- a/include/llama.h
++++ b/include/llama.h
+@@ -444,6 +444,9 @@ extern "C" {
+     // Frees all allocated memory
+     LLAMA_API void llama_free(struct llama_context * ctx);
+ 
++    // save image embeddings
++    LLAMA_API void set_image_embeds(struct llama_context *ctx, float *data);
++
+     LLAMA_API int64_t llama_time_us(void);
+ 
+     LLAMA_API size_t llama_max_devices(void);
+diff --git a/src/llama.cpp b/src/llama.cpp
+index a7b1c9eb..b0a6bc27 100644
+--- a/src/llama.cpp
++++ b/src/llama.cpp
+@@ -2668,6 +2668,7 @@ struct llama_context {
+ 
+     const struct llama_model & model;
+ 
++    float *image_embeds;
+     struct llama_cparams        cparams;
+     struct llama_sampling       sampling;
+     struct llama_kv_cache       kv_self;
+@@ -2751,6 +2752,10 @@ struct llama_context {
+     struct ggml_tensor * inp_KQ_mask_cross; // F32 [n_outputs_enc, n_batch]
+ };
+ 
++void set_image_embeds(llama_context *ctx, float *data) {
++    ctx->image_embeds = data;
++}
++
+ struct llama_lora_weight {
+     struct ggml_tensor * a = nullptr;
+     struct ggml_tensor * b = nullptr;
+@@ -11599,6 +11604,15 @@ struct llm_build_context {
+ 
+         inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
+ 
++        // set the image embeddings in the input tensor
++        if (lctx.image_embeds) {
++            struct ggml_tensor *image_embeds = ggml_dup_tensor(ctx0, inpL);
++            image_embeds->data = lctx.image_embeds;
++            image_embeds->ne[1] = 256;
++            inpL = ggml_set_2d_inplace(ctx0, inpL, image_embeds, inpL->nb[1], 0);
++            lctx.image_embeds = NULL;
++        }
++
+         inpL = ggml_scale(ctx0, inpL, sqrtf(n_embd));
+         cb(inpL, "inp_scaled", -1);
+ 
+@@ -14589,7 +14603,7 @@ static int llama_decode_internal(
+         }
+ 
+         // non-causal masks do not use the KV cache
+-        if (hparams.causal_attn) {
++        if (hparams.causal_attn || lctx.image_embeds) {
+             llama_kv_cache_update(&lctx);
+ 
+             // if we have enough unused cells before the current head ->