diff --git a/cmd/cmd.go b/cmd/cmd.go
index 3bb8b06e..008ca010 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
-	"slices"
 	"strconv"
 	"strings"
 	"sync/atomic"
@@ -453,7 +452,7 @@ func RunHandler(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	opts.MultiModal = slices.Contains(info.Details.Families, "clip")
+	opts.MultiModal = len(info.ProjectorInfo) != 0
 	opts.ParentModel = info.Details.ParentModel
 
 	if interactive {
diff --git a/server/model.go b/server/model.go
index 124693d3..4926d6ce 100644
--- a/server/model.go
+++ b/server/model.go
@@ -194,7 +194,9 @@ func parseFromFile(ctx context.Context, command string, baseLayers []*layerGGML,
 		mediatype := "application/vnd.ollama.image.model"
 		if ggml.Name() == "ggla" || ggml.KV().Kind() == "adapter" {
 			mediatype = "application/vnd.ollama.image.adapter"
-		} else if ggml.KV().Architecture() == "clip" {
+		}
+
+		if _, ok := ggml.KV()[fmt.Sprintf("%s.vision.block_count", ggml.KV().Architecture())]; ok || ggml.KV().Kind() == "projector" {
 			mediatype = "application/vnd.ollama.image.projector"
 		}